{"cells":[{"cell_type":"markdown","id":"6023ea1c","metadata":{"id":"6023ea1c"},"source":["The following additional libraries are needed to run this\n","notebook. Note that running on Colab is experimental, please report a Github\n","issue if you have any problem."]},{"cell_type":"code","execution_count":null,"id":"1b94dc92","metadata":{"id":"1b94dc92","colab":{"base_uri":"https://localhost:8080/","height":1000},"executionInfo":{"status":"ok","timestamp":1663768118852,"user_tz":-480,"elapsed":36952,"user":{"displayName":"Geeks Z","userId":"18417645384289412694"}},"outputId":"355de7a7-250c-4ba8-c5ed-d28a2b1bd807"},"outputs":[{"output_type":"stream","name":"stdout","text":["Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n","Collecting git+https://github.com/d2l-ai/d2l-zh@release\n","  Cloning https://github.com/d2l-ai/d2l-zh (to revision release) to /tmp/pip-req-build-nxm_h6mx\n","  Running command git clone -q https://github.com/d2l-ai/d2l-zh /tmp/pip-req-build-nxm_h6mx\n","  Running command git checkout -b release --track origin/release\n","  Switched to a new branch 'release'\n","  Branch 'release' set up to track remote branch 'release' from 'origin'.\n","  Running command git submodule update --init --recursive -q\n","Collecting jupyter==1.0.0\n","  Downloading jupyter-1.0.0-py2.py3-none-any.whl (2.7 kB)\n","Collecting numpy==1.21.5\n","  Downloading numpy-1.21.5-cp37-cp37m-manylinux_2_12_x86_64.manylinux2010_x86_64.whl (15.7 MB)\n","\u001b[K     |████████████████████████████████| 15.7 MB 4.9 MB/s \n","\u001b[?25hCollecting matplotlib==3.5.1\n","  Downloading matplotlib-3.5.1-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.whl (11.2 MB)\n","\u001b[K     |████████████████████████████████| 11.2 MB 51.7 MB/s \n","\u001b[?25hCollecting requests==2.25.1\n","  Downloading requests-2.25.1-py2.py3-none-any.whl (61 kB)\n","\u001b[K     |████████████████████████████████| 61 kB 9.5 MB/s \n","\u001b[?25hCollecting pandas==1.2.4\n","  Downloading pandas-1.2.4-cp37-cp37m-manylinux1_x86_64.whl (9.9 MB)\n","\u001b[K     |████████████████████████████████| 9.9 MB 57.0 MB/s \n","\u001b[?25hRequirement already satisfied: ipywidgets in /usr/local/lib/python3.7/dist-packages (from jupyter==1.0.0->d2l==2.0.0b1) (7.7.1)\n","Requirement already satisfied: jupyter-console in /usr/local/lib/python3.7/dist-packages (from jupyter==1.0.0->d2l==2.0.0b1) (6.1.0)\n","Requirement already satisfied: nbconvert in /usr/local/lib/python3.7/dist-packages (from jupyter==1.0.0->d2l==2.0.0b1) (5.6.1)\n","Requirement already satisfied: notebook in /usr/local/lib/python3.7/dist-packages (from jupyter==1.0.0->d2l==2.0.0b1) (5.3.1)\n","Collecting qtconsole\n","  Downloading qtconsole-5.3.2-py3-none-any.whl (120 kB)\n","\u001b[K     |████████████████████████████████| 120 kB 76.4 MB/s \n","\u001b[?25hRequirement already satisfied: ipykernel in /usr/local/lib/python3.7/dist-packages (from jupyter==1.0.0->d2l==2.0.0b1) (5.3.4)\n","Requirement already satisfied: python-dateutil>=2.7 in /usr/local/lib/python3.7/dist-packages (from matplotlib==3.5.1->d2l==2.0.0b1) (2.8.2)\n","Requirement already satisfied: cycler>=0.10 in /usr/local/lib/python3.7/dist-packages (from matplotlib==3.5.1->d2l==2.0.0b1) (0.11.0)\n","Requirement already satisfied: pillow>=6.2.0 in /usr/local/lib/python3.7/dist-packages (from matplotlib==3.5.1->d2l==2.0.0b1) (7.1.2)\n","Collecting fonttools>=4.22.0\n","  Downloading fonttools-4.37.3-py3-none-any.whl (959 kB)\n","\u001b[K     |████████████████████████████████| 959 kB 62.8 MB/s \n","\u001b[?25hRequirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.7/dist-packages (from matplotlib==3.5.1->d2l==2.0.0b1) (21.3)\n","Requirement already satisfied: pyparsing>=2.2.1 in /usr/local/lib/python3.7/dist-packages (from matplotlib==3.5.1->d2l==2.0.0b1) (3.0.9)\n","Requirement already satisfied: kiwisolver>=1.0.1 in /usr/local/lib/python3.7/dist-packages (from matplotlib==3.5.1->d2l==2.0.0b1) (1.4.4)\n","Requirement already satisfied: pytz>=2017.3 in /usr/local/lib/python3.7/dist-packages (from pandas==1.2.4->d2l==2.0.0b1) (2022.2.1)\n","Requirement already satisfied: urllib3<1.27,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests==2.25.1->d2l==2.0.0b1) (1.24.3)\n","Requirement already satisfied: chardet<5,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests==2.25.1->d2l==2.0.0b1) (3.0.4)\n","Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests==2.25.1->d2l==2.0.0b1) (2.10)\n","Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests==2.25.1->d2l==2.0.0b1) (2022.6.15)\n","Requirement already satisfied: typing-extensions in /usr/local/lib/python3.7/dist-packages (from kiwisolver>=1.0.1->matplotlib==3.5.1->d2l==2.0.0b1) (4.1.1)\n","Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.7/dist-packages (from python-dateutil>=2.7->matplotlib==3.5.1->d2l==2.0.0b1) (1.15.0)\n","Requirement already satisfied: jupyter-client in /usr/local/lib/python3.7/dist-packages (from ipykernel->jupyter==1.0.0->d2l==2.0.0b1) (6.1.12)\n","Requirement already satisfied: traitlets>=4.1.0 in /usr/local/lib/python3.7/dist-packages (from ipykernel->jupyter==1.0.0->d2l==2.0.0b1) (5.1.1)\n","Requirement already satisfied: ipython>=5.0.0 in /usr/local/lib/python3.7/dist-packages (from ipykernel->jupyter==1.0.0->d2l==2.0.0b1) (7.9.0)\n","Requirement already satisfied: tornado>=4.2 in /usr/local/lib/python3.7/dist-packages (from ipykernel->jupyter==1.0.0->d2l==2.0.0b1) (5.1.1)\n","Requirement already satisfied: pickleshare in /usr/local/lib/python3.7/dist-packages (from ipython>=5.0.0->ipykernel->jupyter==1.0.0->d2l==2.0.0b1) (0.7.5)\n","Collecting jedi>=0.10\n","  Downloading jedi-0.18.1-py2.py3-none-any.whl (1.6 MB)\n","\u001b[K     |████████████████████████████████| 1.6 MB 52.1 MB/s \n","\u001b[?25hRequirement already satisfied: pygments in /usr/local/lib/python3.7/dist-packages (from ipython>=5.0.0->ipykernel->jupyter==1.0.0->d2l==2.0.0b1) (2.6.1)\n","Requirement already satisfied: backcall in /usr/local/lib/python3.7/dist-packages (from ipython>=5.0.0->ipykernel->jupyter==1.0.0->d2l==2.0.0b1) (0.2.0)\n","Requirement already satisfied: pexpect in /usr/local/lib/python3.7/dist-packages (from ipython>=5.0.0->ipykernel->jupyter==1.0.0->d2l==2.0.0b1) (4.8.0)\n","Requirement already satisfied: setuptools>=18.5 in /usr/local/lib/python3.7/dist-packages (from ipython>=5.0.0->ipykernel->jupyter==1.0.0->d2l==2.0.0b1) (57.4.0)\n","Requirement already satisfied: prompt-toolkit<2.1.0,>=2.0.0 in /usr/local/lib/python3.7/dist-packages (from ipython>=5.0.0->ipykernel->jupyter==1.0.0->d2l==2.0.0b1) (2.0.10)\n","Requirement already satisfied: decorator in /usr/local/lib/python3.7/dist-packages (from ipython>=5.0.0->ipykernel->jupyter==1.0.0->d2l==2.0.0b1) (4.4.2)\n","Requirement already satisfied: parso<0.9.0,>=0.8.0 in /usr/local/lib/python3.7/dist-packages (from jedi>=0.10->ipython>=5.0.0->ipykernel->jupyter==1.0.0->d2l==2.0.0b1) (0.8.3)\n","Requirement already satisfied: wcwidth in /usr/local/lib/python3.7/dist-packages (from prompt-toolkit<2.1.0,>=2.0.0->ipython>=5.0.0->ipykernel->jupyter==1.0.0->d2l==2.0.0b1) (0.2.5)\n","Requirement already satisfied: jupyterlab-widgets>=1.0.0 in /usr/local/lib/python3.7/dist-packages (from ipywidgets->jupyter==1.0.0->d2l==2.0.0b1) (3.0.3)\n","Requirement already satisfied: ipython-genutils~=0.2.0 in /usr/local/lib/python3.7/dist-packages (from ipywidgets->jupyter==1.0.0->d2l==2.0.0b1) (0.2.0)\n","Requirement already satisfied: widgetsnbextension~=3.6.0 in /usr/local/lib/python3.7/dist-packages (from ipywidgets->jupyter==1.0.0->d2l==2.0.0b1) (3.6.1)\n","Requirement already satisfied: jinja2 in /usr/local/lib/python3.7/dist-packages (from notebook->jupyter==1.0.0->d2l==2.0.0b1) (2.11.3)\n","Requirement already satisfied: Send2Trash in /usr/local/lib/python3.7/dist-packages (from notebook->jupyter==1.0.0->d2l==2.0.0b1) (1.8.0)\n","Requirement already satisfied: jupyter-core>=4.4.0 in /usr/local/lib/python3.7/dist-packages (from notebook->jupyter==1.0.0->d2l==2.0.0b1) (4.11.1)\n","Requirement already satisfied: terminado>=0.8.1 in /usr/local/lib/python3.7/dist-packages (from notebook->jupyter==1.0.0->d2l==2.0.0b1) (0.13.3)\n","Requirement already satisfied: nbformat in /usr/local/lib/python3.7/dist-packages (from notebook->jupyter==1.0.0->d2l==2.0.0b1) (5.4.0)\n","Requirement already satisfied: pyzmq>=13 in /usr/local/lib/python3.7/dist-packages (from jupyter-client->ipykernel->jupyter==1.0.0->d2l==2.0.0b1) (23.2.1)\n","Requirement already satisfied: ptyprocess in /usr/local/lib/python3.7/dist-packages (from terminado>=0.8.1->notebook->jupyter==1.0.0->d2l==2.0.0b1) (0.7.0)\n","Requirement already satisfied: MarkupSafe>=0.23 in /usr/local/lib/python3.7/dist-packages (from jinja2->notebook->jupyter==1.0.0->d2l==2.0.0b1) (2.0.1)\n","Requirement already satisfied: bleach in /usr/local/lib/python3.7/dist-packages (from nbconvert->jupyter==1.0.0->d2l==2.0.0b1) (5.0.1)\n","Requirement already satisfied: pandocfilters>=1.4.1 in /usr/local/lib/python3.7/dist-packages (from nbconvert->jupyter==1.0.0->d2l==2.0.0b1) (1.5.0)\n","Requirement already satisfied: defusedxml in /usr/local/lib/python3.7/dist-packages (from nbconvert->jupyter==1.0.0->d2l==2.0.0b1) (0.7.1)\n","Requirement already satisfied: mistune<2,>=0.8.1 in /usr/local/lib/python3.7/dist-packages (from nbconvert->jupyter==1.0.0->d2l==2.0.0b1) (0.8.4)\n","Requirement already satisfied: entrypoints>=0.2.2 in /usr/local/lib/python3.7/dist-packages (from nbconvert->jupyter==1.0.0->d2l==2.0.0b1) (0.4)\n","Requirement already satisfied: testpath in /usr/local/lib/python3.7/dist-packages (from nbconvert->jupyter==1.0.0->d2l==2.0.0b1) (0.6.0)\n","Requirement already satisfied: jsonschema>=2.6 in /usr/local/lib/python3.7/dist-packages (from nbformat->notebook->jupyter==1.0.0->d2l==2.0.0b1) (4.3.3)\n","Requirement already satisfied: fastjsonschema in /usr/local/lib/python3.7/dist-packages (from nbformat->notebook->jupyter==1.0.0->d2l==2.0.0b1) (2.16.1)\n","Requirement already satisfied: attrs>=17.4.0 in /usr/local/lib/python3.7/dist-packages (from jsonschema>=2.6->nbformat->notebook->jupyter==1.0.0->d2l==2.0.0b1) (22.1.0)\n","Requirement already satisfied: pyrsistent!=0.17.0,!=0.17.1,!=0.17.2,>=0.14.0 in /usr/local/lib/python3.7/dist-packages (from jsonschema>=2.6->nbformat->notebook->jupyter==1.0.0->d2l==2.0.0b1) (0.18.1)\n","Requirement already satisfied: importlib-metadata in /usr/local/lib/python3.7/dist-packages (from jsonschema>=2.6->nbformat->notebook->jupyter==1.0.0->d2l==2.0.0b1) (4.12.0)\n","Requirement already satisfied: importlib-resources>=1.4.0 in /usr/local/lib/python3.7/dist-packages (from jsonschema>=2.6->nbformat->notebook->jupyter==1.0.0->d2l==2.0.0b1) (5.9.0)\n","Requirement already satisfied: zipp>=3.1.0 in /usr/local/lib/python3.7/dist-packages (from importlib-resources>=1.4.0->jsonschema>=2.6->nbformat->notebook->jupyter==1.0.0->d2l==2.0.0b1) (3.8.1)\n","Requirement already satisfied: webencodings in /usr/local/lib/python3.7/dist-packages (from bleach->nbconvert->jupyter==1.0.0->d2l==2.0.0b1) (0.5.1)\n","Collecting qtpy>=2.0.1\n","  Downloading QtPy-2.2.0-py3-none-any.whl (82 kB)\n","\u001b[K     |████████████████████████████████| 82 kB 634 kB/s \n","\u001b[?25hBuilding wheels for collected packages: d2l\n","  Building wheel for d2l (setup.py) ... \u001b[?25l\u001b[?25hdone\n","  Created wheel for d2l: filename=d2l-2.0.0b1-py3-none-any.whl size=80147 sha256=405c79155ad9e2b2807082bf40dfa3fcb055aa8ff0643099e76c0ef80e07e16f\n","  Stored in directory: /tmp/pip-ephem-wheel-cache-mf9vneph/wheels/73/f4/42/d2b85ca46d85a241d6aa57c1c24027de2d2258202bb67945f9\n","Successfully built d2l\n","Installing collected packages: jedi, qtpy, qtconsole, numpy, fonttools, requests, pandas, matplotlib, jupyter, d2l\n","  Attempting uninstall: numpy\n","    Found existing installation: numpy 1.21.6\n","    Uninstalling numpy-1.21.6:\n","      Successfully uninstalled numpy-1.21.6\n","  Attempting uninstall: requests\n","    Found existing installation: requests 2.23.0\n","    Uninstalling requests-2.23.0:\n","      Successfully uninstalled requests-2.23.0\n","  Attempting uninstall: pandas\n","    Found existing installation: pandas 1.3.5\n","    Uninstalling pandas-1.3.5:\n","      Successfully uninstalled pandas-1.3.5\n","  Attempting uninstall: matplotlib\n","    Found existing installation: matplotlib 3.2.2\n","    Uninstalling matplotlib-3.2.2:\n","      Successfully uninstalled matplotlib-3.2.2\n","Successfully installed d2l-2.0.0b1 fonttools-4.37.3 jedi-0.18.1 jupyter-1.0.0 matplotlib-3.5.1 numpy-1.21.5 pandas-1.2.4 qtconsole-5.3.2 qtpy-2.2.0 requests-2.25.1\n"]},{"output_type":"display_data","data":{"application/vnd.colab-display-data+json":{"pip_warning":{"packages":["matplotlib","mpl_toolkits","numpy"]}}},"metadata":{}},{"output_type":"stream","name":"stdout","text":["Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n","Collecting git+https://github.com/ipython/matplotlib-inline\n","  Cloning https://github.com/ipython/matplotlib-inline to /tmp/pip-req-build-67pb35nw\n","  Running command git clone -q https://github.com/ipython/matplotlib-inline /tmp/pip-req-build-67pb35nw\n","Requirement already satisfied: traitlets in /usr/local/lib/python3.7/dist-packages (from matplotlib-inline==0.1.6) (5.1.1)\n","Building wheels for collected packages: matplotlib-inline\n","  Building wheel for matplotlib-inline (setup.py) ... \u001b[?25l\u001b[?25hdone\n","  Created wheel for matplotlib-inline: filename=matplotlib_inline-0.1.6-py3-none-any.whl size=9422 sha256=4edd76df2daa06d934b66fab0f41f1fa7f5608080780b181fe0c7ee0abb2cac5\n","  Stored in directory: /tmp/pip-ephem-wheel-cache-kye8s2_d/wheels/8e/4e/9a/dac66ab6df8e9b620e41189fd17327c15fcc2da2aae4d9b50b\n","Successfully built matplotlib-inline\n","Installing collected packages: matplotlib-inline\n","Successfully installed matplotlib-inline-0.1.6\n"]}],"source":["!pip install git+https://github.com/d2l-ai/d2l-zh@release  # installing d2l\n","!pip install git+https://github.com/ipython/matplotlib-inline"]},{"cell_type":"markdown","id":"0152a0d6","metadata":{"origin_pos":0,"id":"0152a0d6"},"source":["# 深度卷积神经网络（AlexNet）\n","\n","\n","在LeNet提出后，卷积神经网络在计算机视觉和机器学习领域中很有名气。但卷积神经网络并没有主导这些领域。这是因为虽然LeNet在小数据集上取得了很好的效果，但是在更大、更真实的数据集上训练卷积神经网络的性能和可行性还有待研究。事实上，在上世纪90年代初到2012年之间的大部分时间里，神经网络往往被其他机器学习方法超越，如支持向量机（support vector machines）。\n","\n","在计算机视觉中，直接将神经网络与其他机器学习方法进行比较也许不公平。这是因为，卷积神经网络的输入是由原始像素值或是经过简单预处理（例如居中、缩放）的像素值组成的。但在使用传统机器学习方法时，从业者永远不会将原始像素作为输入。在传统机器学习方法中，计算机视觉流水线是由经过人的手工精心设计的特征流水线组成的。对于这些传统方法，大部分的进展都来自于对特征有了更聪明的想法，并且学习到的算法往往归于事后的解释。\n","\n","虽然上世纪90年代就有了一些神经网络加速卡，但仅靠它们还不足以开发出有大量参数的深层多通道多层卷积神经网络。此外，当时的数据集仍然相对较小。除了这些障碍，训练神经网络的一些关键技巧仍然缺失，包括启发式参数初始化、随机梯度下降的变体、非挤压激活函数和有效的正则化技术。\n","\n","因此，与训练*端到端*（从像素到分类结果）系统不同，经典机器学习的流水线看起来更像下面这样：\n","\n","1. 获取一个有趣的数据集。在早期，收集这些数据集需要昂贵的传感器（在当时最先进的图像也就100万像素）。\n","2. 根据光学、几何学、其他知识以及偶然的发现，手工对特征数据集进行预处理。\n","3. 通过标准的特征提取算法，如SIFT（尺度不变特征变换） :cite:`Lowe.2004`和SURF（加速鲁棒特征） :cite:`Bay.Tuytelaars.Van-Gool.2006`或其他手动调整的流水线来输入数据。\n","4. 将提取的特征送入最喜欢的分类器中（例如线性模型或其它核方法），以训练分类器。\n","\n","如果你和机器学习研究人员交谈，你会发现他们相信机器学习既重要又美丽：优雅的理论去证明各种模型的性质。机器学习是一个正在蓬勃发展、严谨且非常有用的领域。然而，如果你和计算机视觉研究人员交谈，你会听到一个完全不同的故事。他们会告诉你图像识别的诡异事实————推动领域进步的是数据特征，而不是学习算法。计算机视觉研究人员相信，从对最终模型精度的影响来说，更大或更干净的数据集、或是稍微改进的特征提取，比任何学习算法带来的进步要大得多。\n","\n","## 学习表征\n","\n","另一种预测这个领域发展的方法————观察图像特征的提取方法。在2012年前，图像特征都是机械地计算出来的。事实上，设计一套新的特征函数、改进结果，并撰写论文是盛极一时的潮流。SIFT :cite:`Lowe.2004`、SURF :cite:`Bay.Tuytelaars.Van-Gool.2006`、HOG（定向梯度直方图） :cite:`Dalal.Triggs.2005`、[bags of visual words](https://en.wikipedia.org/wiki/Bag-of-words_model_in_computer_vision)和类似的特征提取方法占据了主导地位。\n","\n","另一组研究人员，包括Yann LeCun、Geoff Hinton、Yoshua Bengio、Andrew Ng、Shun ichi Amari和Juergen Schmidhuber，想法则与众不同：他们认为特征本身应该被学习。此外，他们还认为，在合理地复杂性前提下，特征应该由多个共同学习的神经网络层组成，每个层都有可学习的参数。在机器视觉中，最底层可能检测边缘、颜色和纹理。事实上，Alex Krizhevsky、Ilya Sutskever和Geoff Hinton提出了一种新的卷积神经网络变体*AlexNet*。在2012年ImageNet挑战赛中取得了轰动一时的成绩。AlexNet以Alex Krizhevsky的名字命名，他是论文 :cite:`Krizhevsky.Sutskever.Hinton.2012`的第一作者。\n","\n","有趣的是，在网络的最底层，模型学习到了一些类似于传统滤波器的特征抽取器。 :numref:`fig_filters`是从AlexNet论文 :cite:`Krizhevsky.Sutskever.Hinton.2012`复制的，描述了底层图像特征。\n","\n","![AlexNet第一层学习到的特征抽取器。](https://github.com/d2l-ai/d2l-zh-pytorch-colab/blob/master/img/filters.png?raw=1)\n",":width:`400px`\n",":label:`fig_filters`\n","\n","AlexNet的更高层建立在这些底层表示的基础上，以表示更大的特征，如眼睛、鼻子、草叶等等。而更高的层可以检测整个物体，如人、飞机、狗或飞盘。最终的隐藏神经元可以学习图像的综合表示，从而使属于不同类别的数据易于区分。尽管一直有一群执着的研究者不断钻研，试图学习视觉数据的逐级表征，然而很长一段时间里这些尝试都未有突破。深度卷积神经网络的突破出现在2012年。突破可归因于两个关键因素。\n","\n","### 缺少的成分：数据\n","\n","包含许多特征的深度模型需要大量的有标签数据，才能显著优于基于凸优化的传统方法（如线性方法和核方法）。\n","然而，限于早期计算机有限的存储和90年代有限的研究预算，大部分研究只基于小的公开数据集。例如，不少研究论文基于加州大学欧文分校（UCI）提供的若干个公开数据集，其中许多数据集只有几百至几千张在非自然环境下以低分辨率拍摄的图像。这一状况在2010年前后兴起的大数据浪潮中得到改善。2009年，ImageNet数据集发布，并发起ImageNet挑战赛：要求研究人员从100万个样本中训练模型，以区分1000个不同类别的对象。ImageNet数据集由斯坦福教授李飞飞小组的研究人员开发，利用谷歌图像搜索（Google Image Search）对每一类图像进行预筛选，并利用亚马逊众包（Amazon Mechanical Turk）来标注每张图片的相关类别。这种规模是前所未有的。这项被称为ImageNet的挑战赛推动了计算机视觉和机器学习研究的发展，挑战研究人员确定哪些模型能够在更大的数据规模下表现最好。\n","\n","### 缺少的成分：硬件\n","\n","深度学习对计算资源要求很高，训练可能需要数百个迭代轮数，每次迭代都需要通过代价高昂的许多线性代数层传递数据。这也是为什么在20世纪90年代至21世纪初，优化凸目标的简单算法是研究人员的首选。然而，用GPU训练神经网络改变了这一格局。*图形处理器*（Graphics Processing Unit，GPU）早年用来加速图形处理，使电脑游戏玩家受益。GPU可优化高吞吐量的$4 \\times 4$矩阵和向量乘法，从而服务于基本的图形任务。幸运的是，这些数学运算与卷积层的计算惊人地相似。由此，英伟达（NVIDIA）和ATI已经开始为通用计算操作优化gpu，甚至把它们作为*通用GPU*（general-purpose GPUs，GPGPU）来销售。\n","\n","那么GPU比CPU强在哪里呢？\n","\n","首先，我们深度理解一下中央处理器（Central Processing Unit，CPU）的*核心*。\n","CPU的每个核心都拥有高时钟频率的运行能力，和高达数MB的三级缓存（L3Cache）。\n","它们非常适合执行各种指令，具有分支预测器、深层流水线和其他使CPU能够运行各种程序的功能。\n","然而，这种明显的优势也是它的致命弱点：通用核心的制造成本非常高。\n","它们需要大量的芯片面积、复杂的支持结构（内存接口、内核之间的缓存逻辑、高速互连等等），而且它们在任何单个任务上的性能都相对较差。\n","现代笔记本电脑最多有4核，即使是高端服务器也很少超过64核，因为它们的性价比不高。\n","\n","相比于CPU，GPU由$100 \\sim 1000$个小的处理单元组成（NVIDIA、ATI、ARM和其他芯片供应商之间的细节稍有不同），通常被分成更大的组（NVIDIA称之为warps）。\n","虽然每个GPU核心都相对较弱，有时甚至以低于1GHz的时钟频率运行，但庞大的核心数量使GPU比CPU快几个数量级。\n","例如，NVIDIA最近一代的Ampere GPU架构为每个芯片提供了高达312 TFlops的浮点性能，而CPU的浮点性能到目前为止还没有超过1 TFlops。\n","之所以有如此大的差距，原因其实很简单：首先，功耗往往会随时钟频率呈二次方增长。\n","对于一个CPU核心，假设它的运行速度比GPU快4倍，你可以使用16个GPU内核取代，那么GPU的综合性能就是CPU的$16 \\times 1/4 = 4$倍。\n","其次，GPU内核要简单得多，这使得它们更节能。\n","此外，深度学习中的许多操作需要相对较高的内存带宽，而GPU拥有10倍于CPU的带宽。\n","\n","回到2012年的重大突破，当Alex Krizhevsky和Ilya Sutskever实现了可以在GPU硬件上运行的深度卷积神经网络时，一个重大突破出现了。他们意识到卷积神经网络中的计算瓶颈：卷积和矩阵乘法，都是可以在硬件上并行化的操作。\n","于是，他们使用两个显存为3GB的NVIDIA GTX580 GPU实现了快速卷积运算。他们的创新[cuda-convnet](https://code.google.com/archive/p/cuda-convnet/)几年来它一直是行业标准，并推动了深度学习热潮。\n","\n","## AlexNet\n","\n","2012年，AlexNet横空出世。它首次证明了学习到的特征可以超越手工设计的特征。它一举打破了计算机视觉研究的现状。\n","AlexNet使用了8层卷积神经网络，并以很大的优势赢得了2012年ImageNet图像识别挑战赛。\n","\n","AlexNet和LeNet的架构非常相似，如 所示。\n","注意，这里我们提供了一个稍微精简版本的AlexNet，去除了当年需要两个小型GPU同时运算的设计特点。\n","\n","![从LeNet（左）到AlexNet（右）](http://d2l.ai/_images/alexnet.svg)\n","\n","\n","AlexNet和LeNet的设计理念非常相似，但也存在显著差异。\n","首先，AlexNet比相对较小的LeNet5要深得多。\n","AlexNet由八层组成：五个卷积层、两个全连接隐藏层和一个全连接输出层。\n","其次，AlexNet使用ReLU而不是sigmoid作为其激活函数。\n","下面，让我们深入研究AlexNet的细节。\n","\n","### 模型设计\n","\n","在AlexNet的第一层，卷积窗口的形状是$11\\times11$。\n","由于ImageNet中大多数图像的宽和高比MNIST图像的多10倍以上，因此，需要一个更大的卷积窗口来捕获目标。\n","第二层中的卷积窗口形状被缩减为$5\\times5$，然后是$3\\times3$。\n","此外，在第一层、第二层和第五层卷积层之后，加入窗口形状为$3\\times3$、步幅为2的最大汇聚层。\n","而且，AlexNet的卷积通道数目是LeNet的10倍。\n","\n","在最后一个卷积层后有两个全连接层，分别有4096个输出。\n","这两个巨大的全连接层拥有将近1GB的模型参数。\n","由于早期GPU显存有限，原版的AlexNet采用了双数据流设计，使得每个GPU只负责存储和计算模型的一半参数。\n","幸运的是，现在GPU显存相对充裕，所以我们现在很少需要跨GPU分解模型（因此，我们的AlexNet模型在这方面与原始论文稍有不同）。\n","\n","### 激活函数\n","\n","此外，AlexNet将sigmoid激活函数改为更简单的ReLU激活函数。\n","一方面，ReLU激活函数的计算更简单，它不需要如sigmoid激活函数那般复杂的求幂运算。\n","另一方面，当使用不同的参数初始化方法时，ReLU激活函数使训练模型更加容易。\n","当sigmoid激活函数的输出非常接近于0或1时，这些区域的梯度几乎为0，因此反向传播无法继续更新一些模型参数。\n","相反，ReLU激活函数在正区间的梯度总是1。\n","因此，如果模型参数没有正确初始化，sigmoid函数可能在正区间内得到几乎为0的梯度，从而使模型无法得到有效的训练。\n","\n","### 容量控制和预处理\n","\n","AlexNet通过暂退法控制全连接层的模型复杂度，而LeNet只使用了权重衰减。\n","为了进一步扩充数据，AlexNet在训练时增加了大量的图像增强数据，如翻转、裁切和变色。\n","这使得模型更健壮，更大的样本量有效地减少了过拟合。\n","我们将更详细地讨论数据扩增。\n"]},{"cell_type":"code","execution_count":null,"id":"125b9008","metadata":{"execution":{"iopub.execute_input":"2022-07-31T03:00:12.744307Z","iopub.status.busy":"2022-07-31T03:00:12.744068Z","iopub.status.idle":"2022-07-31T03:00:15.431917Z","shell.execute_reply":"2022-07-31T03:00:15.431199Z"},"origin_pos":2,"tab":["pytorch"],"id":"125b9008"},"outputs":[],"source":["import torch\n","from torch import nn\n","from d2l import torch as d2l\n","\n","net = nn.Sequential(\n","    # 这里，我们使用一个11*11的更大窗口来捕捉对象。\n","    # 同时，步幅为4，以减少输出的高度和宽度。\n","    # 另外，输出通道的数目远大于LeNet\n","    nn.Conv2d(1, 96, kernel_size=11, stride=4, padding=1), nn.ReLU(),\n","    nn.MaxPool2d(kernel_size=3, stride=2),\n","    # 减小卷积窗口，使用填充为2来使得输入与输出的高和宽一致，且增大输出通道数\n","    nn.Conv2d(96, 256, kernel_size=5, padding=2), nn.ReLU(),\n","    nn.MaxPool2d(kernel_size=3, stride=2),\n","    # 使用三个连续的卷积层和较小的卷积窗口。\n","    # 除了最后的卷积层，输出通道的数量进一步增加。\n","    # 在前两个卷积层之后，汇聚层不用于减少输入的高度和宽度\n","    nn.Conv2d(256, 384, kernel_size=3, padding=1), nn.ReLU(),\n","    nn.Conv2d(384, 384, kernel_size=3, padding=1), nn.ReLU(),\n","    nn.Conv2d(384, 256, kernel_size=3, padding=1), nn.ReLU(),\n","    nn.MaxPool2d(kernel_size=3, stride=2),\n","    nn.Flatten(),\n","    # 这里，全连接层的输出数量是LeNet中的好几倍。使用dropout层来减轻过拟合\n","    nn.Linear(6400, 4096), nn.ReLU(),\n","    nn.Dropout(p=0.5),\n","    nn.Linear(4096, 4096), nn.ReLU(),\n","    nn.Dropout(p=0.5),\n","    # 最后是输出层。由于这里使用Fashion-MNIST，所以用类别数为10，而非论文中的1000\n","    nn.Linear(4096, 10))"]},{"cell_type":"markdown","id":"699da7ef","metadata":{"origin_pos":4,"id":"699da7ef"},"source":["我们构造一个高度和宽度都为224的单通道数据，来观察每一层输出的形状。\n"]},{"cell_type":"code","execution_count":null,"id":"f4e44ede","metadata":{"execution":{"iopub.execute_input":"2022-07-31T03:00:15.435715Z","iopub.status.busy":"2022-07-31T03:00:15.435209Z","iopub.status.idle":"2022-07-31T03:00:15.507269Z","shell.execute_reply":"2022-07-31T03:00:15.506555Z"},"origin_pos":6,"tab":["pytorch"],"id":"f4e44ede","outputId":"ca814d1e-be9b-4b35-da53-afea7c54ab4c","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1663768358145,"user_tz":-480,"elapsed":418,"user":{"displayName":"Geeks Z","userId":"18417645384289412694"}}},"outputs":[{"output_type":"stream","name":"stdout","text":["Conv2d output shape:\t torch.Size([1, 96, 54, 54])\n","ReLU output shape:\t torch.Size([1, 96, 54, 54])\n","MaxPool2d output shape:\t torch.Size([1, 96, 26, 26])\n","Conv2d output shape:\t torch.Size([1, 256, 26, 26])\n","ReLU output shape:\t torch.Size([1, 256, 26, 26])\n","MaxPool2d output shape:\t torch.Size([1, 256, 12, 12])\n","Conv2d output shape:\t torch.Size([1, 384, 12, 12])\n","ReLU output shape:\t torch.Size([1, 384, 12, 12])\n","Conv2d output shape:\t torch.Size([1, 384, 12, 12])\n","ReLU output shape:\t torch.Size([1, 384, 12, 12])\n","Conv2d output shape:\t torch.Size([1, 256, 12, 12])\n","ReLU output shape:\t torch.Size([1, 256, 12, 12])\n","MaxPool2d output shape:\t torch.Size([1, 256, 5, 5])\n","Flatten output shape:\t torch.Size([1, 6400])\n","Linear output shape:\t torch.Size([1, 4096])\n","ReLU output shape:\t torch.Size([1, 4096])\n","Dropout output shape:\t torch.Size([1, 4096])\n","Linear output shape:\t torch.Size([1, 4096])\n","ReLU output shape:\t torch.Size([1, 4096])\n","Dropout output shape:\t torch.Size([1, 4096])\n","Linear output shape:\t torch.Size([1, 10])\n"]}],"source":["X = torch.randn(1, 1, 224, 224)\n","for layer in net:\n","    X=layer(X)\n","    print(layer.__class__.__name__,'output shape:\\t',X.shape)"]},{"cell_type":"markdown","id":"be06b67b","metadata":{"origin_pos":8,"id":"be06b67b"},"source":["## 读取数据集\n","\n","尽管本文中AlexNet是在ImageNet上进行训练的，但我们在这里使用的是Fashion-MNIST数据集。因为即使在现代GPU上，训练ImageNet模型，同时使其收敛可能需要数小时或数天的时间。\n","将AlexNet直接应用于Fashion-MNIST的一个问题是，[**Fashion-MNIST图像的分辨率**]（$28 \\times 28$像素）(**低于ImageNet图像。**)\n","为了解决这个问题，(**我们将它们增加到$224 \\times 224$**)（通常来讲这不是一个明智的做法，但我们在这里这样做是为了有效使用AlexNet架构）。\n","我们使用`d2l.load_data_fashion_mnist`函数中的`resize`参数执行此调整。\n"]},{"cell_type":"code","execution_count":null,"id":"a1c0ca21","metadata":{"execution":{"iopub.execute_input":"2022-07-31T03:00:15.512962Z","iopub.status.busy":"2022-07-31T03:00:15.512468Z","iopub.status.idle":"2022-07-31T03:00:15.600207Z","shell.execute_reply":"2022-07-31T03:00:15.599495Z"},"origin_pos":9,"tab":["pytorch"],"id":"a1c0ca21","colab":{"base_uri":"https://localhost:8080/","height":486,"referenced_widgets":["d5822e13bd314e51b6dee0f82cdec61f","a7ad688df25b4d04a4d9c254433393cf","f479f8d0e9794d098cf36181ea86ae49","a7d34739a47049bf9311db55d0d601f1","49f17da285724ae7886148ed10d84625","e661161a211842abaac7bcf1e7147112","53df30ce3cf74550ab5798a04428fb6c","1aee2a1bcd6741ac92acceb8386a8277","3661470a06df4c5ca5998fe03bf76643","0ec39731080e4eec82d0616a4fdcf81f","98e33315f8404232b8d6a676bb17623a","d2801dada8074b2395d512eaa91a148d","325d3ef0ea954f339bf9986af34d836c","3704aa63de714107be0aba2d505494f4","fab456d0a0004eee825ece8b6cb9ad2f","4b20325b29664dd5bed17caa47ac7326","62378cf9a46047e19c9a4c4e87a44c06","235ff7d308a84e96aa631ef07e4a1c8c","deaf95c84ff84f519aee7f69fc0a0508","11269d1cef7942f6919519cc176b770d","178e85fa7eb54940950d04c8c51c67c5","a2dcf75791514fdfa22fdf1a470b5b34","f9122b9104894b3eb5cb26add5bb6bc6","e3c2a9ed2d094510b4b897b3e2d655d4","abbead1db37e43a9ad4de89ea245cb1b","e69eab9ed265451c908fe6bdb09ad92f","eee1c312a7b2404bbea1a8d35caaacab","7988d1085b854da6834c44e4a0b10dab","cb8e7ab661664e06b1bd90db506aa872","735c2ebec0f648a9b1d695bd4aaf4439","f92d960292a54e63966f7deec79204ac","669e619890104bd38f07e5c521d01ffe","7f6f3188f91049d8b905b94e4f8acff6","1c88aff658e7429eab4387c443505863","9f68a6adea7f40419b5c7bc864006399","9dbbb787d6e045c9854c757b0d14aed8","fd47a473f22a4127ba06f59be431951b","017ed40d528b4724bd3911e635bf93fd","ccfae2cf427844bba2539c636c476193","8b1d0938a3024a93a2f5b52456e28c0f","7a2c2b565ed84942acde681fb6e6b328","58dfd7d60f794cdf82a991794c10e91f","568ccb51478244c1922d231f52c235a5","80211233886b43bdabcf0c853926efff"]},"executionInfo":{"status":"ok","timestamp":1663768424777,"user_tz":-480,"elapsed":5426,"user":{"displayName":"Geeks Z","userId":"18417645384289412694"}},"outputId":"3aac36c6-a5b5-437a-961a-eee1b3f61e60"},"outputs":[{"output_type":"stream","name":"stdout","text":["Downloading http://fashion-mnist.s3-website.eu-central-1.amazonaws.com/train-images-idx3-ubyte.gz\n","Downloading http://fashion-mnist.s3-website.eu-central-1.amazonaws.com/train-images-idx3-ubyte.gz to ../data/FashionMNIST/raw/train-images-idx3-ubyte.gz\n"]},{"output_type":"display_data","data":{"text/plain":["  0%|          | 0/26421880 [00:00<?, ?it/s]"],"application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"d5822e13bd314e51b6dee0f82cdec61f"}},"metadata":{}},{"output_type":"stream","name":"stdout","text":["Extracting ../data/FashionMNIST/raw/train-images-idx3-ubyte.gz to ../data/FashionMNIST/raw\n","\n","Downloading http://fashion-mnist.s3-website.eu-central-1.amazonaws.com/train-labels-idx1-ubyte.gz\n","Downloading http://fashion-mnist.s3-website.eu-central-1.amazonaws.com/train-labels-idx1-ubyte.gz to ../data/FashionMNIST/raw/train-labels-idx1-ubyte.gz\n"]},{"output_type":"display_data","data":{"text/plain":["  0%|          | 0/29515 [00:00<?, ?it/s]"],"application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"d2801dada8074b2395d512eaa91a148d"}},"metadata":{}},{"output_type":"stream","name":"stdout","text":["Extracting ../data/FashionMNIST/raw/train-labels-idx1-ubyte.gz to ../data/FashionMNIST/raw\n","\n","Downloading http://fashion-mnist.s3-website.eu-central-1.amazonaws.com/t10k-images-idx3-ubyte.gz\n","Downloading http://fashion-mnist.s3-website.eu-central-1.amazonaws.com/t10k-images-idx3-ubyte.gz to ../data/FashionMNIST/raw/t10k-images-idx3-ubyte.gz\n"]},{"output_type":"display_data","data":{"text/plain":["  0%|          | 0/4422102 [00:00<?, ?it/s]"],"application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"f9122b9104894b3eb5cb26add5bb6bc6"}},"metadata":{}},{"output_type":"stream","name":"stdout","text":["Extracting ../data/FashionMNIST/raw/t10k-images-idx3-ubyte.gz to ../data/FashionMNIST/raw\n","\n","Downloading http://fashion-mnist.s3-website.eu-central-1.amazonaws.com/t10k-labels-idx1-ubyte.gz\n","Downloading http://fashion-mnist.s3-website.eu-central-1.amazonaws.com/t10k-labels-idx1-ubyte.gz to ../data/FashionMNIST/raw/t10k-labels-idx1-ubyte.gz\n"]},{"output_type":"display_data","data":{"text/plain":["  0%|          | 0/5148 [00:00<?, ?it/s]"],"application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"1c88aff658e7429eab4387c443505863"}},"metadata":{}},{"output_type":"stream","name":"stdout","text":["Extracting ../data/FashionMNIST/raw/t10k-labels-idx1-ubyte.gz to ../data/FashionMNIST/raw\n","\n"]},{"output_type":"stream","name":"stderr","text":["/usr/local/lib/python3.7/dist-packages/torch/utils/data/dataloader.py:566: UserWarning: This DataLoader will create 4 worker processes in total. Our suggested max number of worker in current system is 2, which is smaller than what this DataLoader is going to create. Please be aware that excessive worker creation might get DataLoader running slow or even freeze, lower the worker number to avoid potential slowness/freeze if necessary.\n","  cpuset_checked))\n"]}],"source":["batch_size = 128\n","train_iter, test_iter = d2l.load_data_fashion_mnist(batch_size, resize=224)"]},{"cell_type":"markdown","id":"ca599e7c","metadata":{"origin_pos":10,"id":"ca599e7c"},"source":["## [**训练AlexNet**]\n","\n","现在，我们可以开始训练AlexNet了。与 :numref:`sec_lenet`中的LeNet相比，这里的主要变化是使用更小的学习速率训练，这是因为网络更深更广、图像分辨率更高，训练卷积神经网络就更昂贵。\n"]},{"cell_type":"code","execution_count":null,"id":"e02388fd","metadata":{"execution":{"iopub.execute_input":"2022-07-31T03:00:15.605077Z","iopub.status.busy":"2022-07-31T03:00:15.604696Z","iopub.status.idle":"2022-07-31T03:03:48.199878Z","shell.execute_reply":"2022-07-31T03:03:48.199184Z"},"origin_pos":11,"tab":["pytorch"],"id":"e02388fd","outputId":"8c234d99-f8e4-4a76-9ee5-72f8a3138432","colab":{"base_uri":"https://localhost:8080/","height":297},"executionInfo":{"status":"ok","timestamp":1663768982115,"user_tz":-480,"elapsed":529864,"user":{"displayName":"Geeks Z","userId":"18417645384289412694"}}},"outputs":[{"output_type":"stream","name":"stdout","text":["loss 0.331, train acc 0.879, test acc 0.873\n","1441.3 examples/sec on cuda:0\n"]},{"output_type":"display_data","data":{"text/plain":["<Figure size 252x180 with 1 Axes>"],"image/svg+xml":"<?xml version=\"1.0\" encoding=\"utf-8\" standalone=\"no\"?>\n<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\n  \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\n<svg xmlns:xlink=\"http://www.w3.org/1999/xlink\" width=\"238.965625pt\" height=\"180.65625pt\" viewBox=\"0 0 238.965625 180.65625\" xmlns=\"http://www.w3.org/2000/svg\" version=\"1.1\">\n <metadata>\n  <rdf:RDF xmlns:dc=\"http://purl.org/dc/elements/1.1/\" xmlns:cc=\"http://creativecommons.org/ns#\" xmlns:rdf=\"http://www.w3.org/1999/02/22-rdf-syntax-ns#\">\n   <cc:Work>\n    <dc:type rdf:resource=\"http://purl.org/dc/dcmitype/StillImage\"/>\n    <dc:date>2022-09-21T14:03:01.979597</dc:date>\n    <dc:format>image/svg+xml</dc:format>\n    <dc:creator>\n     <cc:Agent>\n      <dc:title>Matplotlib v3.5.1, https://matplotlib.org/</dc:title>\n     </cc:Agent>\n    </dc:creator>\n   </cc:Work>\n  </rdf:RDF>\n </metadata>\n <defs>\n  <style type=\"text/css\">*{stroke-linejoin: round; stroke-linecap: butt}</style>\n </defs>\n <g id=\"figure_1\">\n  <g id=\"patch_1\">\n   <path d=\"M 0 180.65625 \nL 238.965625 180.65625 \nL 238.965625 0 \nL 0 0 \nL 0 180.65625 \nz\n\" style=\"fill: none\"/>\n  </g>\n  <g id=\"axes_1\">\n   <g id=\"patch_2\">\n    <path d=\"M 30.103125 143.1 \nL 225.403125 143.1 \nL 225.403125 7.2 \nL 30.103125 7.2 \nz\n\" style=\"fill: #ffffff\"/>\n   </g>\n   <g id=\"matplotlib.axis_1\">\n    <g id=\"xtick_1\">\n     <g id=\"line2d_1\">\n      <path d=\"M 51.803125 143.1 \nL 51.803125 7.2 \n\" clip-path=\"url(#pa100d383a8)\" style=\"fill: none; stroke: #b0b0b0; stroke-width: 0.8; stroke-linecap: square\"/>\n     </g>\n     <g id=\"line2d_2\">\n      <defs>\n       <path id=\"m08e1cee453\" d=\"M 0 0 \nL 0 3.5 \n\" style=\"stroke: #000000; stroke-width: 0.8\"/>\n      </defs>\n      <g>\n       <use xlink:href=\"#m08e1cee453\" x=\"51.803125\" y=\"143.1\" style=\"stroke: #000000; stroke-width: 0.8\"/>\n      </g>\n     </g>\n     <g id=\"text_1\">\n      <!-- 2 -->\n      <g transform=\"translate(48.621875 157.698438)scale(0.1 -0.1)\">\n       <defs>\n        <path id=\"DejaVuSans-32\" d=\"M 1228 531 \nL 3431 531 \nL 3431 0 \nL 469 0 \nL 469 531 \nQ 828 903 1448 1529 \nQ 2069 2156 2228 2338 \nQ 2531 2678 2651 2914 \nQ 2772 3150 2772 3378 \nQ 2772 3750 2511 3984 \nQ 2250 4219 1831 4219 \nQ 1534 4219 1204 4116 \nQ 875 4013 500 3803 \nL 500 4441 \nQ 881 4594 1212 4672 \nQ 1544 4750 1819 4750 \nQ 2544 4750 2975 4387 \nQ 3406 4025 3406 3419 \nQ 3406 3131 3298 2873 \nQ 3191 2616 2906 2266 \nQ 2828 2175 2409 1742 \nQ 1991 1309 1228 531 \nz\n\" transform=\"scale(0.015625)\"/>\n       </defs>\n       <use xlink:href=\"#DejaVuSans-32\"/>\n      </g>\n     </g>\n    </g>\n    <g id=\"xtick_2\">\n     <g id=\"line2d_3\">\n      <path d=\"M 95.203125 143.1 \nL 95.203125 7.2 \n\" clip-path=\"url(#pa100d383a8)\" style=\"fill: none; stroke: #b0b0b0; stroke-width: 0.8; stroke-linecap: square\"/>\n     </g>\n     <g id=\"line2d_4\">\n      <g>\n       <use xlink:href=\"#m08e1cee453\" x=\"95.203125\" y=\"143.1\" style=\"stroke: #000000; stroke-width: 0.8\"/>\n      </g>\n     </g>\n     <g id=\"text_2\">\n      <!-- 4 -->\n      <g transform=\"translate(92.021875 157.698438)scale(0.1 -0.1)\">\n       <defs>\n        <path id=\"DejaVuSans-34\" d=\"M 2419 4116 \nL 825 1625 \nL 2419 1625 \nL 2419 4116 \nz\nM 2253 4666 \nL 3047 4666 \nL 3047 1625 \nL 3713 1625 \nL 3713 1100 \nL 3047 1100 \nL 3047 0 \nL 2419 0 \nL 2419 1100 \nL 313 1100 \nL 313 1709 \nL 2253 4666 \nz\n\" transform=\"scale(0.015625)\"/>\n       </defs>\n       <use xlink:href=\"#DejaVuSans-34\"/>\n      </g>\n     </g>\n    </g>\n    <g id=\"xtick_3\">\n     <g id=\"line2d_5\">\n      <path d=\"M 138.603125 143.1 \nL 138.603125 7.2 \n\" clip-path=\"url(#pa100d383a8)\" style=\"fill: none; stroke: #b0b0b0; stroke-width: 0.8; stroke-linecap: square\"/>\n     </g>\n     <g id=\"line2d_6\">\n      <g>\n       <use xlink:href=\"#m08e1cee453\" x=\"138.603125\" y=\"143.1\" style=\"stroke: #000000; stroke-width: 0.8\"/>\n      </g>\n     </g>\n     <g id=\"text_3\">\n      <!-- 6 -->\n      <g transform=\"translate(135.421875 157.698438)scale(0.1 -0.1)\">\n       <defs>\n        <path id=\"DejaVuSans-36\" d=\"M 2113 2584 \nQ 1688 2584 1439 2293 \nQ 1191 2003 1191 1497 \nQ 1191 994 1439 701 \nQ 1688 409 2113 409 \nQ 2538 409 2786 701 \nQ 3034 994 3034 1497 \nQ 3034 2003 2786 2293 \nQ 2538 2584 2113 2584 \nz\nM 3366 4563 \nL 3366 3988 \nQ 3128 4100 2886 4159 \nQ 2644 4219 2406 4219 \nQ 1781 4219 1451 3797 \nQ 1122 3375 1075 2522 \nQ 1259 2794 1537 2939 \nQ 1816 3084 2150 3084 \nQ 2853 3084 3261 2657 \nQ 3669 2231 3669 1497 \nQ 3669 778 3244 343 \nQ 2819 -91 2113 -91 \nQ 1303 -91 875 529 \nQ 447 1150 447 2328 \nQ 447 3434 972 4092 \nQ 1497 4750 2381 4750 \nQ 2619 4750 2861 4703 \nQ 3103 4656 3366 4563 \nz\n\" transform=\"scale(0.015625)\"/>\n       </defs>\n       <use xlink:href=\"#DejaVuSans-36\"/>\n      </g>\n     </g>\n    </g>\n    <g id=\"xtick_4\">\n     <g id=\"line2d_7\">\n      <path d=\"M 182.003125 143.1 \nL 182.003125 7.2 \n\" clip-path=\"url(#pa100d383a8)\" style=\"fill: none; stroke: #b0b0b0; stroke-width: 0.8; stroke-linecap: square\"/>\n     </g>\n     <g id=\"line2d_8\">\n      <g>\n       <use xlink:href=\"#m08e1cee453\" x=\"182.003125\" y=\"143.1\" style=\"stroke: #000000; stroke-width: 0.8\"/>\n      </g>\n     </g>\n     <g id=\"text_4\">\n      <!-- 8 -->\n      <g transform=\"translate(178.821875 157.698438)scale(0.1 -0.1)\">\n       <defs>\n        <path id=\"DejaVuSans-38\" d=\"M 2034 2216 \nQ 1584 2216 1326 1975 \nQ 1069 1734 1069 1313 \nQ 1069 891 1326 650 \nQ 1584 409 2034 409 \nQ 2484 409 2743 651 \nQ 3003 894 3003 1313 \nQ 3003 1734 2745 1975 \nQ 2488 2216 2034 2216 \nz\nM 1403 2484 \nQ 997 2584 770 2862 \nQ 544 3141 544 3541 \nQ 544 4100 942 4425 \nQ 1341 4750 2034 4750 \nQ 2731 4750 3128 4425 \nQ 3525 4100 3525 3541 \nQ 3525 3141 3298 2862 \nQ 3072 2584 2669 2484 \nQ 3125 2378 3379 2068 \nQ 3634 1759 3634 1313 \nQ 3634 634 3220 271 \nQ 2806 -91 2034 -91 \nQ 1263 -91 848 271 \nQ 434 634 434 1313 \nQ 434 1759 690 2068 \nQ 947 2378 1403 2484 \nz\nM 1172 3481 \nQ 1172 3119 1398 2916 \nQ 1625 2713 2034 2713 \nQ 2441 2713 2670 2916 \nQ 2900 3119 2900 3481 \nQ 2900 3844 2670 4047 \nQ 2441 4250 2034 4250 \nQ 1625 4250 1398 4047 \nQ 1172 3844 1172 3481 \nz\n\" transform=\"scale(0.015625)\"/>\n       </defs>\n       <use xlink:href=\"#DejaVuSans-38\"/>\n      </g>\n     </g>\n    </g>\n    <g id=\"xtick_5\">\n     <g id=\"line2d_9\">\n      <path d=\"M 225.403125 143.1 \nL 225.403125 7.2 \n\" clip-path=\"url(#pa100d383a8)\" style=\"fill: none; stroke: #b0b0b0; stroke-width: 0.8; stroke-linecap: square\"/>\n     </g>\n     <g id=\"line2d_10\">\n      <g>\n       <use xlink:href=\"#m08e1cee453\" x=\"225.403125\" y=\"143.1\" style=\"stroke: #000000; stroke-width: 0.8\"/>\n      </g>\n     </g>\n     <g id=\"text_5\">\n      <!-- 10 -->\n      <g transform=\"translate(219.040625 157.698438)scale(0.1 -0.1)\">\n       <defs>\n        <path id=\"DejaVuSans-31\" d=\"M 794 531 \nL 1825 531 \nL 1825 4091 \nL 703 3866 \nL 703 4441 \nL 1819 4666 \nL 2450 4666 \nL 2450 531 \nL 3481 531 \nL 3481 0 \nL 794 0 \nL 794 531 \nz\n\" transform=\"scale(0.015625)\"/>\n        <path id=\"DejaVuSans-30\" d=\"M 2034 4250 \nQ 1547 4250 1301 3770 \nQ 1056 3291 1056 2328 \nQ 1056 1369 1301 889 \nQ 1547 409 2034 409 \nQ 2525 409 2770 889 \nQ 3016 1369 3016 2328 \nQ 3016 3291 2770 3770 \nQ 2525 4250 2034 4250 \nz\nM 2034 4750 \nQ 2819 4750 3233 4129 \nQ 3647 3509 3647 2328 \nQ 3647 1150 3233 529 \nQ 2819 -91 2034 -91 \nQ 1250 -91 836 529 \nQ 422 1150 422 2328 \nQ 422 3509 836 4129 \nQ 1250 4750 2034 4750 \nz\n\" transform=\"scale(0.015625)\"/>\n       </defs>\n       <use xlink:href=\"#DejaVuSans-31\"/>\n       <use xlink:href=\"#DejaVuSans-30\" x=\"63.623047\"/>\n      </g>\n     </g>\n    </g>\n    <g id=\"text_6\">\n     <!-- epoch -->\n     <g transform=\"translate(112.525 171.376563)scale(0.1 -0.1)\">\n      <defs>\n       <path id=\"DejaVuSans-65\" d=\"M 3597 1894 \nL 3597 1613 \nL 953 1613 \nQ 991 1019 1311 708 \nQ 1631 397 2203 397 \nQ 2534 397 2845 478 \nQ 3156 559 3463 722 \nL 3463 178 \nQ 3153 47 2828 -22 \nQ 2503 -91 2169 -91 \nQ 1331 -91 842 396 \nQ 353 884 353 1716 \nQ 353 2575 817 3079 \nQ 1281 3584 2069 3584 \nQ 2775 3584 3186 3129 \nQ 3597 2675 3597 1894 \nz\nM 3022 2063 \nQ 3016 2534 2758 2815 \nQ 2500 3097 2075 3097 \nQ 1594 3097 1305 2825 \nQ 1016 2553 972 2059 \nL 3022 2063 \nz\n\" transform=\"scale(0.015625)\"/>\n       <path id=\"DejaVuSans-70\" d=\"M 1159 525 \nL 1159 -1331 \nL 581 -1331 \nL 581 3500 \nL 1159 3500 \nL 1159 2969 \nQ 1341 3281 1617 3432 \nQ 1894 3584 2278 3584 \nQ 2916 3584 3314 3078 \nQ 3713 2572 3713 1747 \nQ 3713 922 3314 415 \nQ 2916 -91 2278 -91 \nQ 1894 -91 1617 61 \nQ 1341 213 1159 525 \nz\nM 3116 1747 \nQ 3116 2381 2855 2742 \nQ 2594 3103 2138 3103 \nQ 1681 3103 1420 2742 \nQ 1159 2381 1159 1747 \nQ 1159 1113 1420 752 \nQ 1681 391 2138 391 \nQ 2594 391 2855 752 \nQ 3116 1113 3116 1747 \nz\n\" transform=\"scale(0.015625)\"/>\n       <path id=\"DejaVuSans-6f\" d=\"M 1959 3097 \nQ 1497 3097 1228 2736 \nQ 959 2375 959 1747 \nQ 959 1119 1226 758 \nQ 1494 397 1959 397 \nQ 2419 397 2687 759 \nQ 2956 1122 2956 1747 \nQ 2956 2369 2687 2733 \nQ 2419 3097 1959 3097 \nz\nM 1959 3584 \nQ 2709 3584 3137 3096 \nQ 3566 2609 3566 1747 \nQ 3566 888 3137 398 \nQ 2709 -91 1959 -91 \nQ 1206 -91 779 398 \nQ 353 888 353 1747 \nQ 353 2609 779 3096 \nQ 1206 3584 1959 3584 \nz\n\" transform=\"scale(0.015625)\"/>\n       <path id=\"DejaVuSans-63\" d=\"M 3122 3366 \nL 3122 2828 \nQ 2878 2963 2633 3030 \nQ 2388 3097 2138 3097 \nQ 1578 3097 1268 2742 \nQ 959 2388 959 1747 \nQ 959 1106 1268 751 \nQ 1578 397 2138 397 \nQ 2388 397 2633 464 \nQ 2878 531 3122 666 \nL 3122 134 \nQ 2881 22 2623 -34 \nQ 2366 -91 2075 -91 \nQ 1284 -91 818 406 \nQ 353 903 353 1747 \nQ 353 2603 823 3093 \nQ 1294 3584 2113 3584 \nQ 2378 3584 2631 3529 \nQ 2884 3475 3122 3366 \nz\n\" transform=\"scale(0.015625)\"/>\n       <path id=\"DejaVuSans-68\" d=\"M 3513 2113 \nL 3513 0 \nL 2938 0 \nL 2938 2094 \nQ 2938 2591 2744 2837 \nQ 2550 3084 2163 3084 \nQ 1697 3084 1428 2787 \nQ 1159 2491 1159 1978 \nL 1159 0 \nL 581 0 \nL 581 4863 \nL 1159 4863 \nL 1159 2956 \nQ 1366 3272 1645 3428 \nQ 1925 3584 2291 3584 \nQ 2894 3584 3203 3211 \nQ 3513 2838 3513 2113 \nz\n\" transform=\"scale(0.015625)\"/>\n      </defs>\n      <use xlink:href=\"#DejaVuSans-65\"/>\n      <use xlink:href=\"#DejaVuSans-70\" x=\"61.523438\"/>\n      <use xlink:href=\"#DejaVuSans-6f\" x=\"125\"/>\n      <use xlink:href=\"#DejaVuSans-63\" x=\"186.181641\"/>\n      <use xlink:href=\"#DejaVuSans-68\" x=\"241.162109\"/>\n     </g>\n    </g>\n   </g>\n   <g id=\"matplotlib.axis_2\">\n    <g id=\"ytick_1\">\n     <g id=\"line2d_11\">\n      <path d=\"M 30.103125 117.272229 \nL 225.403125 117.272229 \n\" clip-path=\"url(#pa100d383a8)\" style=\"fill: none; stroke: #b0b0b0; stroke-width: 0.8; stroke-linecap: square\"/>\n     </g>\n     <g id=\"line2d_12\">\n      <defs>\n       <path id=\"m7221ed0a1b\" d=\"M 0 0 \nL -3.5 0 \n\" style=\"stroke: #000000; stroke-width: 0.8\"/>\n      </defs>\n      <g>\n       <use xlink:href=\"#m7221ed0a1b\" x=\"30.103125\" y=\"117.272229\" style=\"stroke: #000000; stroke-width: 0.8\"/>\n      </g>\n     </g>\n     <g id=\"text_7\">\n      <!-- 0.5 -->\n      <g transform=\"translate(7.2 121.071448)scale(0.1 -0.1)\">\n       <defs>\n        <path id=\"DejaVuSans-2e\" d=\"M 684 794 \nL 1344 794 \nL 1344 0 \nL 684 0 \nL 684 794 \nz\n\" transform=\"scale(0.015625)\"/>\n        <path id=\"DejaVuSans-35\" d=\"M 691 4666 \nL 3169 4666 \nL 3169 4134 \nL 1269 4134 \nL 1269 2991 \nQ 1406 3038 1543 3061 \nQ 1681 3084 1819 3084 \nQ 2600 3084 3056 2656 \nQ 3513 2228 3513 1497 \nQ 3513 744 3044 326 \nQ 2575 -91 1722 -91 \nQ 1428 -91 1123 -41 \nQ 819 9 494 109 \nL 494 744 \nQ 775 591 1075 516 \nQ 1375 441 1709 441 \nQ 2250 441 2565 725 \nQ 2881 1009 2881 1497 \nQ 2881 1984 2565 2268 \nQ 2250 2553 1709 2553 \nQ 1456 2553 1204 2497 \nQ 953 2441 691 2322 \nL 691 4666 \nz\n\" transform=\"scale(0.015625)\"/>\n       </defs>\n       <use xlink:href=\"#DejaVuSans-30\"/>\n       <use xlink:href=\"#DejaVuSans-2e\" x=\"63.623047\"/>\n       <use xlink:href=\"#DejaVuSans-35\" x=\"95.410156\"/>\n      </g>\n     </g>\n    </g>\n    <g id=\"ytick_2\">\n     <g id=\"line2d_13\">\n      <path d=\"M 30.103125 88.083369 \nL 225.403125 88.083369 \n\" clip-path=\"url(#pa100d383a8)\" style=\"fill: none; stroke: #b0b0b0; stroke-width: 0.8; stroke-linecap: square\"/>\n     </g>\n     <g id=\"line2d_14\">\n      <g>\n       <use xlink:href=\"#m7221ed0a1b\" x=\"30.103125\" y=\"88.083369\" style=\"stroke: #000000; stroke-width: 0.8\"/>\n      </g>\n     </g>\n     <g id=\"text_8\">\n      <!-- 1.0 -->\n      <g transform=\"translate(7.2 91.882588)scale(0.1 -0.1)\">\n       <use xlink:href=\"#DejaVuSans-31\"/>\n       <use xlink:href=\"#DejaVuSans-2e\" x=\"63.623047\"/>\n       <use xlink:href=\"#DejaVuSans-30\" x=\"95.410156\"/>\n      </g>\n     </g>\n    </g>\n    <g id=\"ytick_3\">\n     <g id=\"line2d_15\">\n      <path d=\"M 30.103125 58.894509 \nL 225.403125 58.894509 \n\" clip-path=\"url(#pa100d383a8)\" style=\"fill: none; stroke: #b0b0b0; stroke-width: 0.8; stroke-linecap: square\"/>\n     </g>\n     <g id=\"line2d_16\">\n      <g>\n       <use xlink:href=\"#m7221ed0a1b\" x=\"30.103125\" y=\"58.894509\" style=\"stroke: #000000; stroke-width: 0.8\"/>\n      </g>\n     </g>\n     <g id=\"text_9\">\n      <!-- 1.5 -->\n      <g transform=\"translate(7.2 62.693727)scale(0.1 -0.1)\">\n       <use xlink:href=\"#DejaVuSans-31\"/>\n       <use xlink:href=\"#DejaVuSans-2e\" x=\"63.623047\"/>\n       <use xlink:href=\"#DejaVuSans-35\" x=\"95.410156\"/>\n      </g>\n     </g>\n    </g>\n    <g id=\"ytick_4\">\n     <g id=\"line2d_17\">\n      <path d=\"M 30.103125 29.705648 \nL 225.403125 29.705648 \n\" clip-path=\"url(#pa100d383a8)\" style=\"fill: none; stroke: #b0b0b0; stroke-width: 0.8; stroke-linecap: square\"/>\n     </g>\n     <g id=\"line2d_18\">\n      <g>\n       <use xlink:href=\"#m7221ed0a1b\" x=\"30.103125\" y=\"29.705648\" style=\"stroke: #000000; stroke-width: 0.8\"/>\n      </g>\n     </g>\n     <g id=\"text_10\">\n      <!-- 2.0 -->\n      <g transform=\"translate(7.2 33.504867)scale(0.1 -0.1)\">\n       <use xlink:href=\"#DejaVuSans-32\"/>\n       <use xlink:href=\"#DejaVuSans-2e\" x=\"63.623047\"/>\n       <use xlink:href=\"#DejaVuSans-30\" x=\"95.410156\"/>\n      </g>\n     </g>\n    </g>\n   </g>\n   <g id=\"line2d_19\">\n    <path d=\"M 12.70611 13.377273 \nL 17.009095 32.993398 \nL 21.31208 50.786924 \nL 25.615065 61.815856 \nL 29.91805 69.803582 \nL 30.103125 70.073459 \nL 34.40611 105.133264 \nL 38.709095 106.223445 \nL 43.01208 107.156873 \nL 47.315065 107.99815 \nL 51.61805 108.953715 \nL 51.803125 108.953589 \nL 56.10611 113.09409 \nL 60.409095 114.312457 \nL 64.71208 114.736199 \nL 69.015065 115.078264 \nL 73.31805 115.396479 \nL 73.503125 115.417109 \nL 77.80611 117.226865 \nL 82.109095 118.019419 \nL 86.41208 118.341369 \nL 90.715065 118.672318 \nL 95.01805 118.906643 \nL 95.203125 118.920006 \nL 99.50611 119.83389 \nL 103.809095 120.429481 \nL 108.11208 121.071581 \nL 112.415065 121.150585 \nL 116.71805 121.360665 \nL 116.903125 121.383429 \nL 121.20611 122.450282 \nL 125.509095 122.665268 \nL 129.81208 122.84426 \nL 134.115065 123.052849 \nL 138.41805 123.120345 \nL 138.603125 123.137924 \nL 142.90611 124.666091 \nL 147.209095 124.603617 \nL 151.51208 124.501611 \nL 155.815065 124.553689 \nL 160.11805 124.403022 \nL 160.303125 124.405731 \nL 164.60611 125.534579 \nL 168.909095 125.33128 \nL 173.21208 125.312056 \nL 177.515065 125.454794 \nL 181.81805 125.484282 \nL 182.003125 125.489151 \nL 186.30611 125.523822 \nL 190.609095 125.808877 \nL 194.91208 126.116072 \nL 199.215065 126.241156 \nL 203.51805 126.396826 \nL 203.703125 126.426893 \nL 208.00611 126.743423 \nL 212.309095 127.151241 \nL 216.61208 127.342965 \nL 220.915065 127.205706 \nL 225.21805 127.179542 \nL 225.403125 127.140003 \n\" clip-path=\"url(#pa100d383a8)\" style=\"fill: none; stroke: #1f77b4; stroke-width: 1.5; stroke-linecap: square\"/>\n   </g>\n   <g id=\"line2d_20\">\n    <path d=\"M 12.70611 136.922727 \nL 17.009095 129.669649 \nL 21.31208 123.470939 \nL 25.615065 119.442268 \nL 29.91805 116.54545 \nL 30.103125 116.443266 \nL 34.40611 103.570335 \nL 38.709095 103.136327 \nL 43.01208 102.770976 \nL 47.315065 102.436275 \nL 51.61805 102.119719 \nL 51.803125 102.124184 \nL 56.10611 100.745606 \nL 60.409095 100.306695 \nL 64.71208 100.08683 \nL 69.015065 99.938891 \nL 73.31805 99.783433 \nL 73.503125 99.773508 \nL 77.80611 99.225353 \nL 82.109095 98.82077 \nL 86.41208 98.615617 \nL 90.715065 98.42722 \nL 95.01805 98.329875 \nL 95.203125 98.328659 \nL 99.50611 98.11704 \nL 103.809095 97.844865 \nL 108.11208 97.577595 \nL 112.415065 97.48687 \nL 116.71805 97.375548 \nL 116.903125 97.371264 \nL 121.20611 96.768428 \nL 125.509095 96.712032 \nL 129.81208 96.639288 \nL 134.115065 96.615177 \nL 138.41805 96.592863 \nL 138.603125 96.593868 \nL 142.90611 96.062246 \nL 147.209095 96.04263 \nL 151.51208 96.11946 \nL 155.815065 96.074506 \nL 160.11805 96.123056 \nL 160.303125 96.1249 \nL 164.60611 95.733675 \nL 168.909095 95.81214 \nL 173.21208 95.841564 \nL 177.515065 95.742257 \nL 181.81805 95.756234 \nL 182.003125 95.761985 \nL 186.30611 95.576746 \nL 190.609095 95.466405 \nL 194.91208 95.421451 \nL 199.215065 95.448015 \nL 203.51805 95.390392 \nL 203.703125 95.386422 \nL 208.00611 95.243271 \nL 212.309095 95.150094 \nL 216.61208 95.066725 \nL 220.915065 95.172162 \nL 225.21805 95.150094 \nL 225.403125 95.157776 \n\" clip-path=\"url(#pa100d383a8)\" style=\"fill: none; stroke-dasharray: 5.55,2.4; stroke-dashoffset: 0; stroke: #bf00bf; stroke-width: 1.5\"/>\n   </g>\n   <g id=\"line2d_21\">\n    <path d=\"M 30.103125 102.43845 \nL 51.803125 99.840642 \nL 73.503125 98.77233 \nL 95.203125 97.902502 \nL 116.903125 96.606516 \nL 138.603125 96.629867 \nL 160.303125 95.900146 \nL 182.003125 95.643284 \nL 203.703125 95.473988 \nL 225.403125 95.497339 \n\" clip-path=\"url(#pa100d383a8)\" style=\"fill: none; stroke-dasharray: 9.6,2.4,1.5,2.4; stroke-dashoffset: 0; stroke: #008000; stroke-width: 1.5\"/>\n   </g>\n   <g id=\"patch_3\">\n    <path d=\"M 30.103125 143.1 \nL 30.103125 7.2 \n\" style=\"fill: none; stroke: #000000; stroke-width: 0.8; stroke-linejoin: miter; stroke-linecap: square\"/>\n   </g>\n   <g id=\"patch_4\">\n    <path d=\"M 225.403125 143.1 \nL 225.403125 7.2 \n\" style=\"fill: none; stroke: #000000; stroke-width: 0.8; stroke-linejoin: miter; stroke-linecap: square\"/>\n   </g>\n   <g id=\"patch_5\">\n    <path d=\"M 30.103125 143.1 \nL 225.403125 143.1 \n\" style=\"fill: none; stroke: #000000; stroke-width: 0.8; stroke-linejoin: miter; stroke-linecap: square\"/>\n   </g>\n   <g id=\"patch_6\">\n    <path d=\"M 30.103125 7.2 \nL 225.403125 7.2 \n\" style=\"fill: none; stroke: #000000; stroke-width: 0.8; stroke-linejoin: miter; stroke-linecap: square\"/>\n   </g>\n   <g id=\"legend_1\">\n    <g id=\"patch_7\">\n     <path d=\"M 140.634375 59.234375 \nL 218.403125 59.234375 \nQ 220.403125 59.234375 220.403125 57.234375 \nL 220.403125 14.2 \nQ 220.403125 12.2 218.403125 12.2 \nL 140.634375 12.2 \nQ 138.634375 12.2 138.634375 14.2 \nL 138.634375 57.234375 \nQ 138.634375 59.234375 140.634375 59.234375 \nz\n\" style=\"fill: #ffffff; opacity: 0.8; stroke: #cccccc; stroke-linejoin: miter\"/>\n    </g>\n    <g id=\"line2d_22\">\n     <path d=\"M 142.634375 20.298438 \nL 152.634375 20.298438 \nL 162.634375 20.298438 \n\" style=\"fill: none; stroke: #1f77b4; stroke-width: 1.5; stroke-linecap: square\"/>\n    </g>\n    <g id=\"text_11\">\n     <!-- train loss -->\n     <g transform=\"translate(170.634375 23.798438)scale(0.1 -0.1)\">\n      <defs>\n       <path id=\"DejaVuSans-74\" d=\"M 1172 4494 \nL 1172 3500 \nL 2356 3500 \nL 2356 3053 \nL 1172 3053 \nL 1172 1153 \nQ 1172 725 1289 603 \nQ 1406 481 1766 481 \nL 2356 481 \nL 2356 0 \nL 1766 0 \nQ 1100 0 847 248 \nQ 594 497 594 1153 \nL 594 3053 \nL 172 3053 \nL 172 3500 \nL 594 3500 \nL 594 4494 \nL 1172 4494 \nz\n\" transform=\"scale(0.015625)\"/>\n       <path id=\"DejaVuSans-72\" d=\"M 2631 2963 \nQ 2534 3019 2420 3045 \nQ 2306 3072 2169 3072 \nQ 1681 3072 1420 2755 \nQ 1159 2438 1159 1844 \nL 1159 0 \nL 581 0 \nL 581 3500 \nL 1159 3500 \nL 1159 2956 \nQ 1341 3275 1631 3429 \nQ 1922 3584 2338 3584 \nQ 2397 3584 2469 3576 \nQ 2541 3569 2628 3553 \nL 2631 2963 \nz\n\" transform=\"scale(0.015625)\"/>\n       <path id=\"DejaVuSans-61\" d=\"M 2194 1759 \nQ 1497 1759 1228 1600 \nQ 959 1441 959 1056 \nQ 959 750 1161 570 \nQ 1363 391 1709 391 \nQ 2188 391 2477 730 \nQ 2766 1069 2766 1631 \nL 2766 1759 \nL 2194 1759 \nz\nM 3341 1997 \nL 3341 0 \nL 2766 0 \nL 2766 531 \nQ 2569 213 2275 61 \nQ 1981 -91 1556 -91 \nQ 1019 -91 701 211 \nQ 384 513 384 1019 \nQ 384 1609 779 1909 \nQ 1175 2209 1959 2209 \nL 2766 2209 \nL 2766 2266 \nQ 2766 2663 2505 2880 \nQ 2244 3097 1772 3097 \nQ 1472 3097 1187 3025 \nQ 903 2953 641 2809 \nL 641 3341 \nQ 956 3463 1253 3523 \nQ 1550 3584 1831 3584 \nQ 2591 3584 2966 3190 \nQ 3341 2797 3341 1997 \nz\n\" transform=\"scale(0.015625)\"/>\n       <path id=\"DejaVuSans-69\" d=\"M 603 3500 \nL 1178 3500 \nL 1178 0 \nL 603 0 \nL 603 3500 \nz\nM 603 4863 \nL 1178 4863 \nL 1178 4134 \nL 603 4134 \nL 603 4863 \nz\n\" transform=\"scale(0.015625)\"/>\n       <path id=\"DejaVuSans-6e\" d=\"M 3513 2113 \nL 3513 0 \nL 2938 0 \nL 2938 2094 \nQ 2938 2591 2744 2837 \nQ 2550 3084 2163 3084 \nQ 1697 3084 1428 2787 \nQ 1159 2491 1159 1978 \nL 1159 0 \nL 581 0 \nL 581 3500 \nL 1159 3500 \nL 1159 2956 \nQ 1366 3272 1645 3428 \nQ 1925 3584 2291 3584 \nQ 2894 3584 3203 3211 \nQ 3513 2838 3513 2113 \nz\n\" transform=\"scale(0.015625)\"/>\n       <path id=\"DejaVuSans-20\" transform=\"scale(0.015625)\"/>\n       <path id=\"DejaVuSans-6c\" d=\"M 603 4863 \nL 1178 4863 \nL 1178 0 \nL 603 0 \nL 603 4863 \nz\n\" transform=\"scale(0.015625)\"/>\n       <path id=\"DejaVuSans-73\" d=\"M 2834 3397 \nL 2834 2853 \nQ 2591 2978 2328 3040 \nQ 2066 3103 1784 3103 \nQ 1356 3103 1142 2972 \nQ 928 2841 928 2578 \nQ 928 2378 1081 2264 \nQ 1234 2150 1697 2047 \nL 1894 2003 \nQ 2506 1872 2764 1633 \nQ 3022 1394 3022 966 \nQ 3022 478 2636 193 \nQ 2250 -91 1575 -91 \nQ 1294 -91 989 -36 \nQ 684 19 347 128 \nL 347 722 \nQ 666 556 975 473 \nQ 1284 391 1588 391 \nQ 1994 391 2212 530 \nQ 2431 669 2431 922 \nQ 2431 1156 2273 1281 \nQ 2116 1406 1581 1522 \nL 1381 1569 \nQ 847 1681 609 1914 \nQ 372 2147 372 2553 \nQ 372 3047 722 3315 \nQ 1072 3584 1716 3584 \nQ 2034 3584 2315 3537 \nQ 2597 3491 2834 3397 \nz\n\" transform=\"scale(0.015625)\"/>\n      </defs>\n      <use xlink:href=\"#DejaVuSans-74\"/>\n      <use xlink:href=\"#DejaVuSans-72\" x=\"39.208984\"/>\n      <use xlink:href=\"#DejaVuSans-61\" x=\"80.322266\"/>\n      <use xlink:href=\"#DejaVuSans-69\" x=\"141.601562\"/>\n      <use xlink:href=\"#DejaVuSans-6e\" x=\"169.384766\"/>\n      <use xlink:href=\"#DejaVuSans-20\" x=\"232.763672\"/>\n      <use xlink:href=\"#DejaVuSans-6c\" x=\"264.550781\"/>\n      <use xlink:href=\"#DejaVuSans-6f\" x=\"292.333984\"/>\n      <use xlink:href=\"#DejaVuSans-73\" x=\"353.515625\"/>\n      <use xlink:href=\"#DejaVuSans-73\" x=\"405.615234\"/>\n     </g>\n    </g>\n    <g id=\"line2d_23\">\n     <path d=\"M 142.634375 34.976562 \nL 152.634375 34.976562 \nL 162.634375 34.976562 \n\" style=\"fill: none; stroke-dasharray: 5.55,2.4; stroke-dashoffset: 0; stroke: #bf00bf; stroke-width: 1.5\"/>\n    </g>\n    <g id=\"text_12\">\n     <!-- train acc -->\n     <g transform=\"translate(170.634375 38.476562)scale(0.1 -0.1)\">\n      <use xlink:href=\"#DejaVuSans-74\"/>\n      <use xlink:href=\"#DejaVuSans-72\" x=\"39.208984\"/>\n      <use xlink:href=\"#DejaVuSans-61\" x=\"80.322266\"/>\n      <use xlink:href=\"#DejaVuSans-69\" x=\"141.601562\"/>\n      <use xlink:href=\"#DejaVuSans-6e\" x=\"169.384766\"/>\n      <use xlink:href=\"#DejaVuSans-20\" x=\"232.763672\"/>\n      <use xlink:href=\"#DejaVuSans-61\" x=\"264.550781\"/>\n      <use xlink:href=\"#DejaVuSans-63\" x=\"325.830078\"/>\n      <use xlink:href=\"#DejaVuSans-63\" x=\"380.810547\"/>\n     </g>\n    </g>\n    <g id=\"line2d_24\">\n     <path d=\"M 142.634375 49.654688 \nL 152.634375 49.654688 \nL 162.634375 49.654688 \n\" style=\"fill: none; stroke-dasharray: 9.6,2.4,1.5,2.4; stroke-dashoffset: 0; stroke: #008000; stroke-width: 1.5\"/>\n    </g>\n    <g id=\"text_13\">\n     <!-- test acc -->\n     <g transform=\"translate(170.634375 53.154688)scale(0.1 -0.1)\">\n      <use xlink:href=\"#DejaVuSans-74\"/>\n      <use xlink:href=\"#DejaVuSans-65\" x=\"39.208984\"/>\n      <use xlink:href=\"#DejaVuSans-73\" x=\"100.732422\"/>\n      <use xlink:href=\"#DejaVuSans-74\" x=\"152.832031\"/>\n      <use xlink:href=\"#DejaVuSans-20\" x=\"192.041016\"/>\n      <use xlink:href=\"#DejaVuSans-61\" x=\"223.828125\"/>\n      <use xlink:href=\"#DejaVuSans-63\" x=\"285.107422\"/>\n      <use xlink:href=\"#DejaVuSans-63\" x=\"340.087891\"/>\n     </g>\n    </g>\n   </g>\n  </g>\n </g>\n <defs>\n  <clipPath id=\"pa100d383a8\">\n   <rect x=\"30.103125\" y=\"7.2\" width=\"195.3\" height=\"135.9\"/>\n  </clipPath>\n </defs>\n</svg>\n"},"metadata":{"needs_background":"light"}}],"source":["lr, num_epochs = 0.01, 10\n","d2l.train_ch6(net, train_iter, test_iter, num_epochs, lr, d2l.try_gpu())"]},{"cell_type":"markdown","id":"ae904711","metadata":{"origin_pos":12,"id":"ae904711"},"source":["## 小结\n","\n","* AlexNet的架构与LeNet相似，但使用了更多的卷积层和更多的参数来拟合大规模的ImageNet数据集。\n","* 今天，AlexNet已经被更有效的架构所超越，但它是从浅层网络到深层网络的关键一步。\n","* 尽管AlexNet的代码只比LeNet多出几行，但学术界花了很多年才接受深度学习这一概念，并应用其出色的实验结果。这也是由于缺乏有效的计算工具。\n","* Dropout、ReLU和预处理是提升计算机视觉任务性能的其他关键步骤。\n","\n"]},{"cell_type":"markdown","metadata":{"origin_pos":0,"id":"9233a8c1"},"source":["# 使用块的网络（VGG）\n","\n","\n","虽然AlexNet证明深层神经网络卓有成效，但它没有提供一个通用的模板来指导后续的研究人员设计新的网络。\n","在下面的几个章节中，我们将介绍一些常用于设计深层神经网络的启发式概念。\n","\n","与芯片设计中工程师从放置晶体管到逻辑元件再到逻辑块的过程类似，神经网络架构的设计也逐渐变得更加抽象。研究人员开始从单个神经元的角度思考问题，发展到整个层，现在又转向块，重复层的模式。\n","\n","使用块的想法首先出现在牛津大学的[视觉几何组（visualgeometry group）](http://www.robots.ox.ac.uk/~vgg/)的*VGG网络*中。通过使用循环和子程序，可以很容易地在任何现代深度学习框架的代码中实现这些重复的架构。\n","\n","## (**VGG块**)\n","\n","经典卷积神经网络的基本组成部分是下面的这个序列：\n","\n","1. 带填充以保持分辨率的卷积层；\n","1. 非线性激活函数，如ReLU；\n","1. 汇聚层，如最大汇聚层。\n","\n","而一个VGG块与之类似，由一系列卷积层组成，后面再加上用于空间下采样的最大汇聚层。在最初的VGG论文中 ，作者使用了带有$3\\times3$卷积核、填充为1（保持高度和宽度）的卷积层，和带有$2 \\times 2$汇聚窗口、步幅为2（每个块后的分辨率减半）的最大汇聚层。在下面的代码中，我们定义了一个名为`vgg_block`的函数来实现一个VGG块。\n"],"id":"9233a8c1"},{"cell_type":"markdown","metadata":{"origin_pos":2,"tab":["pytorch"],"id":"a1996bd1"},"source":["该函数有三个参数，分别对应于卷积层的数量`num_convs`、输入通道的数量`in_channels`\n","和输出通道的数量`out_channels`.\n"],"id":"a1996bd1"},{"cell_type":"code","execution_count":null,"metadata":{"execution":{"iopub.execute_input":"2022-07-31T03:52:47.721072Z","iopub.status.busy":"2022-07-31T03:52:47.720698Z","iopub.status.idle":"2022-07-31T03:52:51.465067Z","shell.execute_reply":"2022-07-31T03:52:51.464270Z"},"origin_pos":4,"tab":["pytorch"],"id":"b34d2999"},"outputs":[],"source":["import torch\n","from torch import nn\n","from d2l import torch as d2l\n","\n","\n","def vgg_block(num_convs, in_channels, out_channels):\n","    layers = []\n","    for _ in range(num_convs):\n","        layers.append(nn.Conv2d(in_channels, out_channels,\n","                                kernel_size=3, padding=1))\n","        layers.append(nn.ReLU())\n","        in_channels = out_channels\n","    layers.append(nn.MaxPool2d(kernel_size=2,stride=2))\n","    return nn.Sequential(*layers)"],"id":"b34d2999"},{"cell_type":"markdown","metadata":{"origin_pos":6,"id":"84792063"},"source":["## [**VGG网络**]\n","\n","与AlexNet、LeNet一样，VGG网络可以分为两部分：第一部分主要由卷积层和汇聚层组成，第二部分由全连接层组成。如 :numref:`fig_vgg`中所示。\n","\n","![从AlexNet到VGG，它们本质上都是块设计。](http://d2l.ai/_images/vgg.svg)\n",":width:`400px`\n",":label:`fig_vgg`\n","\n","VGG神经网络连接 :numref:`fig_vgg`的几个VGG块（在`vgg_block`函数中定义）。其中有超参数变量`conv_arch`。该变量指定了每个VGG块里卷积层个数和输出通道数。全连接模块则与AlexNet中的相同。\n","\n","原始VGG网络有5个卷积块，其中前两个块各有一个卷积层，后三个块各包含两个卷积层。\n","第一个模块有64个输出通道，每个后续模块将输出通道数量翻倍，直到该数字达到512。由于该网络使用8个卷积层和3个全连接层，因此它通常被称为VGG-11。\n"],"id":"84792063"},{"cell_type":"code","execution_count":null,"metadata":{"execution":{"iopub.execute_input":"2022-07-31T03:52:51.469033Z","iopub.status.busy":"2022-07-31T03:52:51.468465Z","iopub.status.idle":"2022-07-31T03:52:51.472485Z","shell.execute_reply":"2022-07-31T03:52:51.471807Z"},"origin_pos":7,"tab":["pytorch"],"id":"cb7d54c1"},"outputs":[],"source":["conv_arch = ((1, 64), (1, 128), (2, 256), (2, 512), (2, 512))"],"id":"cb7d54c1"},{"cell_type":"markdown","metadata":{"origin_pos":8,"id":"41efaa62"},"source":["下面的代码实现了VGG-11。可以通过在`conv_arch`上执行for循环来简单实现。\n"],"id":"41efaa62"},{"cell_type":"code","execution_count":null,"metadata":{"execution":{"iopub.execute_input":"2022-07-31T03:52:51.475652Z","iopub.status.busy":"2022-07-31T03:52:51.475145Z","iopub.status.idle":"2022-07-31T03:52:52.675313Z","shell.execute_reply":"2022-07-31T03:52:52.674597Z"},"origin_pos":10,"tab":["pytorch"],"id":"9a3fe21d"},"outputs":[],"source":["def vgg(conv_arch):\n","    conv_blks = []\n","    in_channels = 1\n","    # 卷积层部分\n","    for (num_convs, out_channels) in conv_arch:\n","        conv_blks.append(vgg_block(num_convs, in_channels, out_channels))\n","        in_channels = out_channels\n","\n","    return nn.Sequential(\n","        *conv_blks, nn.Flatten(),\n","        # 全连接层部分\n","        nn.Linear(out_channels * 7 * 7, 4096), nn.ReLU(), nn.Dropout(0.5),\n","        nn.Linear(4096, 4096), nn.ReLU(), nn.Dropout(0.5),\n","        nn.Linear(4096, 10))\n","\n","net = vgg(conv_arch)"],"id":"9a3fe21d"},{"cell_type":"markdown","metadata":{"origin_pos":12,"id":"b12a8436"},"source":["接下来，我们将构建一个高度和宽度为224的单通道数据样本，以[**观察每个层输出的形状**]。\n"],"id":"b12a8436"},{"cell_type":"code","execution_count":null,"metadata":{"execution":{"iopub.execute_input":"2022-07-31T03:52:52.678880Z","iopub.status.busy":"2022-07-31T03:52:52.678378Z","iopub.status.idle":"2022-07-31T03:52:52.742694Z","shell.execute_reply":"2022-07-31T03:52:52.741990Z"},"origin_pos":14,"tab":["pytorch"],"id":"e56f99ac","outputId":"67d9aff1-b223-41c5-c638-3899682daddd"},"outputs":[{"name":"stdout","output_type":"stream","text":["Sequential output shape:\t torch.Size([1, 64, 112, 112])\n","Sequential output shape:\t torch.Size([1, 128, 56, 56])\n","Sequential output shape:\t torch.Size([1, 256, 28, 28])\n","Sequential output shape:\t torch.Size([1, 512, 14, 14])\n","Sequential output shape:\t torch.Size([1, 512, 7, 7])\n","Flatten output shape:\t torch.Size([1, 25088])\n","Linear output shape:\t torch.Size([1, 4096])\n","ReLU output shape:\t torch.Size([1, 4096])\n","Dropout output shape:\t torch.Size([1, 4096])\n","Linear output shape:\t torch.Size([1, 4096])\n","ReLU output shape:\t torch.Size([1, 4096])\n","Dropout output shape:\t torch.Size([1, 4096])\n","Linear output shape:\t torch.Size([1, 10])\n"]}],"source":["X = torch.randn(size=(1, 1, 224, 224))\n","for blk in net:\n","    X = blk(X)\n","    print(blk.__class__.__name__,'output shape:\\t',X.shape)"],"id":"e56f99ac"},{"cell_type":"markdown","metadata":{"origin_pos":16,"id":"440e7fe4"},"source":["正如你所看到的，我们在每个块的高度和宽度减半，最终高度和宽度都为7。最后再展平表示，送入全连接层处理。\n","\n","## 训练模型\n","\n","[**由于VGG-11比AlexNet计算量更大，因此我们构建了一个通道数较少的网络**]，足够用于训练Fashion-MNIST数据集。\n"],"id":"440e7fe4"},{"cell_type":"code","execution_count":null,"metadata":{"execution":{"iopub.execute_input":"2022-07-31T03:52:52.748247Z","iopub.status.busy":"2022-07-31T03:52:52.747750Z","iopub.status.idle":"2022-07-31T03:52:53.148450Z","shell.execute_reply":"2022-07-31T03:52:53.147715Z"},"origin_pos":17,"tab":["pytorch"],"id":"51b7046b"},"outputs":[],"source":["ratio = 4\n","small_conv_arch = [(pair[0], pair[1] // ratio) for pair in conv_arch]\n","net = vgg(small_conv_arch)"],"id":"51b7046b"},{"cell_type":"markdown","metadata":{"origin_pos":19,"id":"0712d3e1"},"source":["除了使用略高的学习率外，[**模型训练**]过程与 :numref:`sec_alexnet`中的AlexNet类似。\n"],"id":"0712d3e1"},{"cell_type":"code","execution_count":null,"metadata":{"execution":{"iopub.execute_input":"2022-07-31T03:52:53.153411Z","iopub.status.busy":"2022-07-31T03:52:53.152929Z","iopub.status.idle":"2022-07-31T03:58:00.510821Z","shell.execute_reply":"2022-07-31T03:58:00.509898Z"},"origin_pos":20,"tab":["pytorch"],"id":"c6a88053","outputId":"446bc05f-f78c-4390-f84b-cd5df65fa6b5"},"outputs":[{"name":"stdout","output_type":"stream","text":["loss 0.177, train acc 0.934, test acc 0.911\n","2562.3 examples/sec on cuda:0\n"]},{"data":{"image/svg+xml":["<?xml version=\"1.0\" encoding=\"utf-8\" standalone=\"no\"?>\n","<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\n","  \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\n","<svg xmlns:xlink=\"http://www.w3.org/1999/xlink\" width=\"238.965625pt\" height=\"180.65625pt\" viewBox=\"0 0 238.965625 180.65625\" xmlns=\"http://www.w3.org/2000/svg\" version=\"1.1\">\n"," <metadata>\n","  <rdf:RDF xmlns:dc=\"http://purl.org/dc/elements/1.1/\" xmlns:cc=\"http://creativecommons.org/ns#\" xmlns:rdf=\"http://www.w3.org/1999/02/22-rdf-syntax-ns#\">\n","   <cc:Work>\n","    <dc:type rdf:resource=\"http://purl.org/dc/dcmitype/StillImage\"/>\n","    <dc:date>2022-07-31T03:58:00.469226</dc:date>\n","    <dc:format>image/svg+xml</dc:format>\n","    <dc:creator>\n","     <cc:Agent>\n","      <dc:title>Matplotlib v3.5.1, https://matplotlib.org/</dc:title>\n","     </cc:Agent>\n","    </dc:creator>\n","   </cc:Work>\n","  </rdf:RDF>\n"," </metadata>\n"," <defs>\n","  <style type=\"text/css\">*{stroke-linejoin: round; stroke-linecap: butt}</style>\n"," </defs>\n"," <g id=\"figure_1\">\n","  <g id=\"patch_1\">\n","   <path d=\"M 0 180.65625 \n","L 238.965625 180.65625 \n","L 238.965625 0 \n","L 0 0 \n","L 0 180.65625 \n","z\n","\" style=\"fill: none\"/>\n","  </g>\n","  <g id=\"axes_1\">\n","   <g id=\"patch_2\">\n","    <path d=\"M 30.103125 143.1 \n","L 225.403125 143.1 \n","L 225.403125 7.2 \n","L 30.103125 7.2 \n","z\n","\" style=\"fill: #ffffff\"/>\n","   </g>\n","   <g id=\"matplotlib.axis_1\">\n","    <g id=\"xtick_1\">\n","     <g id=\"line2d_1\">\n","      <path d=\"M 51.803125 143.1 \n","L 51.803125 7.2 \n","\" clip-path=\"url(#p4793444f25)\" style=\"fill: none; stroke: #b0b0b0; stroke-width: 0.8; stroke-linecap: square\"/>\n","     </g>\n","     <g id=\"line2d_2\">\n","      <defs>\n","       <path id=\"m82ab7436aa\" d=\"M 0 0 \n","L 0 3.5 \n","\" style=\"stroke: #000000; stroke-width: 0.8\"/>\n","      </defs>\n","      <g>\n","       <use xlink:href=\"#m82ab7436aa\" x=\"51.803125\" y=\"143.1\" style=\"stroke: #000000; stroke-width: 0.8\"/>\n","      </g>\n","     </g>\n","     <g id=\"text_1\">\n","      <!-- 2 -->\n","      <g transform=\"translate(48.621875 157.698438)scale(0.1 -0.1)\">\n","       <defs>\n","        <path id=\"DejaVuSans-32\" d=\"M 1228 531 \n","L 3431 531 \n","L 3431 0 \n","L 469 0 \n","L 469 531 \n","Q 828 903 1448 1529 \n","Q 2069 2156 2228 2338 \n","Q 2531 2678 2651 2914 \n","Q 2772 3150 2772 3378 \n","Q 2772 3750 2511 3984 \n","Q 2250 4219 1831 4219 \n","Q 1534 4219 1204 4116 \n","Q 875 4013 500 3803 \n","L 500 4441 \n","Q 881 4594 1212 4672 \n","Q 1544 4750 1819 4750 \n","Q 2544 4750 2975 4387 \n","Q 3406 4025 3406 3419 \n","Q 3406 3131 3298 2873 \n","Q 3191 2616 2906 2266 \n","Q 2828 2175 2409 1742 \n","Q 1991 1309 1228 531 \n","z\n","\" transform=\"scale(0.015625)\"/>\n","       </defs>\n","       <use xlink:href=\"#DejaVuSans-32\"/>\n","      </g>\n","     </g>\n","    </g>\n","    <g id=\"xtick_2\">\n","     <g id=\"line2d_3\">\n","      <path d=\"M 95.203125 143.1 \n","L 95.203125 7.2 \n","\" clip-path=\"url(#p4793444f25)\" style=\"fill: none; stroke: #b0b0b0; stroke-width: 0.8; stroke-linecap: square\"/>\n","     </g>\n","     <g id=\"line2d_4\">\n","      <g>\n","       <use xlink:href=\"#m82ab7436aa\" x=\"95.203125\" y=\"143.1\" style=\"stroke: #000000; stroke-width: 0.8\"/>\n","      </g>\n","     </g>\n","     <g id=\"text_2\">\n","      <!-- 4 -->\n","      <g transform=\"translate(92.021875 157.698438)scale(0.1 -0.1)\">\n","       <defs>\n","        <path id=\"DejaVuSans-34\" d=\"M 2419 4116 \n","L 825 1625 \n","L 2419 1625 \n","L 2419 4116 \n","z\n","M 2253 4666 \n","L 3047 4666 \n","L 3047 1625 \n","L 3713 1625 \n","L 3713 1100 \n","L 3047 1100 \n","L 3047 0 \n","L 2419 0 \n","L 2419 1100 \n","L 313 1100 \n","L 313 1709 \n","L 2253 4666 \n","z\n","\" transform=\"scale(0.015625)\"/>\n","       </defs>\n","       <use xlink:href=\"#DejaVuSans-34\"/>\n","      </g>\n","     </g>\n","    </g>\n","    <g id=\"xtick_3\">\n","     <g id=\"line2d_5\">\n","      <path d=\"M 138.603125 143.1 \n","L 138.603125 7.2 \n","\" clip-path=\"url(#p4793444f25)\" style=\"fill: none; stroke: #b0b0b0; stroke-width: 0.8; stroke-linecap: square\"/>\n","     </g>\n","     <g id=\"line2d_6\">\n","      <g>\n","       <use xlink:href=\"#m82ab7436aa\" x=\"138.603125\" y=\"143.1\" style=\"stroke: #000000; stroke-width: 0.8\"/>\n","      </g>\n","     </g>\n","     <g id=\"text_3\">\n","      <!-- 6 -->\n","      <g transform=\"translate(135.421875 157.698438)scale(0.1 -0.1)\">\n","       <defs>\n","        <path id=\"DejaVuSans-36\" d=\"M 2113 2584 \n","Q 1688 2584 1439 2293 \n","Q 1191 2003 1191 1497 \n","Q 1191 994 1439 701 \n","Q 1688 409 2113 409 \n","Q 2538 409 2786 701 \n","Q 3034 994 3034 1497 \n","Q 3034 2003 2786 2293 \n","Q 2538 2584 2113 2584 \n","z\n","M 3366 4563 \n","L 3366 3988 \n","Q 3128 4100 2886 4159 \n","Q 2644 4219 2406 4219 \n","Q 1781 4219 1451 3797 \n","Q 1122 3375 1075 2522 \n","Q 1259 2794 1537 2939 \n","Q 1816 3084 2150 3084 \n","Q 2853 3084 3261 2657 \n","Q 3669 2231 3669 1497 \n","Q 3669 778 3244 343 \n","Q 2819 -91 2113 -91 \n","Q 1303 -91 875 529 \n","Q 447 1150 447 2328 \n","Q 447 3434 972 4092 \n","Q 1497 4750 2381 4750 \n","Q 2619 4750 2861 4703 \n","Q 3103 4656 3366 4563 \n","z\n","\" transform=\"scale(0.015625)\"/>\n","       </defs>\n","       <use xlink:href=\"#DejaVuSans-36\"/>\n","      </g>\n","     </g>\n","    </g>\n","    <g id=\"xtick_4\">\n","     <g id=\"line2d_7\">\n","      <path d=\"M 182.003125 143.1 \n","L 182.003125 7.2 \n","\" clip-path=\"url(#p4793444f25)\" style=\"fill: none; stroke: #b0b0b0; stroke-width: 0.8; stroke-linecap: square\"/>\n","     </g>\n","     <g id=\"line2d_8\">\n","      <g>\n","       <use xlink:href=\"#m82ab7436aa\" x=\"182.003125\" y=\"143.1\" style=\"stroke: #000000; stroke-width: 0.8\"/>\n","      </g>\n","     </g>\n","     <g id=\"text_4\">\n","      <!-- 8 -->\n","      <g transform=\"translate(178.821875 157.698438)scale(0.1 -0.1)\">\n","       <defs>\n","        <path id=\"DejaVuSans-38\" d=\"M 2034 2216 \n","Q 1584 2216 1326 1975 \n","Q 1069 1734 1069 1313 \n","Q 1069 891 1326 650 \n","Q 1584 409 2034 409 \n","Q 2484 409 2743 651 \n","Q 3003 894 3003 1313 \n","Q 3003 1734 2745 1975 \n","Q 2488 2216 2034 2216 \n","z\n","M 1403 2484 \n","Q 997 2584 770 2862 \n","Q 544 3141 544 3541 \n","Q 544 4100 942 4425 \n","Q 1341 4750 2034 4750 \n","Q 2731 4750 3128 4425 \n","Q 3525 4100 3525 3541 \n","Q 3525 3141 3298 2862 \n","Q 3072 2584 2669 2484 \n","Q 3125 2378 3379 2068 \n","Q 3634 1759 3634 1313 \n","Q 3634 634 3220 271 \n","Q 2806 -91 2034 -91 \n","Q 1263 -91 848 271 \n","Q 434 634 434 1313 \n","Q 434 1759 690 2068 \n","Q 947 2378 1403 2484 \n","z\n","M 1172 3481 \n","Q 1172 3119 1398 2916 \n","Q 1625 2713 2034 2713 \n","Q 2441 2713 2670 2916 \n","Q 2900 3119 2900 3481 \n","Q 2900 3844 2670 4047 \n","Q 2441 4250 2034 4250 \n","Q 1625 4250 1398 4047 \n","Q 1172 3844 1172 3481 \n","z\n","\" transform=\"scale(0.015625)\"/>\n","       </defs>\n","       <use xlink:href=\"#DejaVuSans-38\"/>\n","      </g>\n","     </g>\n","    </g>\n","    <g id=\"xtick_5\">\n","     <g id=\"line2d_9\">\n","      <path d=\"M 225.403125 143.1 \n","L 225.403125 7.2 \n","\" clip-path=\"url(#p4793444f25)\" style=\"fill: none; stroke: #b0b0b0; stroke-width: 0.8; stroke-linecap: square\"/>\n","     </g>\n","     <g id=\"line2d_10\">\n","      <g>\n","       <use xlink:href=\"#m82ab7436aa\" x=\"225.403125\" y=\"143.1\" style=\"stroke: #000000; stroke-width: 0.8\"/>\n","      </g>\n","     </g>\n","     <g id=\"text_5\">\n","      <!-- 10 -->\n","      <g transform=\"translate(219.040625 157.698438)scale(0.1 -0.1)\">\n","       <defs>\n","        <path id=\"DejaVuSans-31\" d=\"M 794 531 \n","L 1825 531 \n","L 1825 4091 \n","L 703 3866 \n","L 703 4441 \n","L 1819 4666 \n","L 2450 4666 \n","L 2450 531 \n","L 3481 531 \n","L 3481 0 \n","L 794 0 \n","L 794 531 \n","z\n","\" transform=\"scale(0.015625)\"/>\n","        <path id=\"DejaVuSans-30\" d=\"M 2034 4250 \n","Q 1547 4250 1301 3770 \n","Q 1056 3291 1056 2328 \n","Q 1056 1369 1301 889 \n","Q 1547 409 2034 409 \n","Q 2525 409 2770 889 \n","Q 3016 1369 3016 2328 \n","Q 3016 3291 2770 3770 \n","Q 2525 4250 2034 4250 \n","z\n","M 2034 4750 \n","Q 2819 4750 3233 4129 \n","Q 3647 3509 3647 2328 \n","Q 3647 1150 3233 529 \n","Q 2819 -91 2034 -91 \n","Q 1250 -91 836 529 \n","Q 422 1150 422 2328 \n","Q 422 3509 836 4129 \n","Q 1250 4750 2034 4750 \n","z\n","\" transform=\"scale(0.015625)\"/>\n","       </defs>\n","       <use xlink:href=\"#DejaVuSans-31\"/>\n","       <use xlink:href=\"#DejaVuSans-30\" x=\"63.623047\"/>\n","      </g>\n","     </g>\n","    </g>\n","    <g id=\"text_6\">\n","     <!-- epoch -->\n","     <g transform=\"translate(112.525 171.376563)scale(0.1 -0.1)\">\n","      <defs>\n","       <path id=\"DejaVuSans-65\" d=\"M 3597 1894 \n","L 3597 1613 \n","L 953 1613 \n","Q 991 1019 1311 708 \n","Q 1631 397 2203 397 \n","Q 2534 397 2845 478 \n","Q 3156 559 3463 722 \n","L 3463 178 \n","Q 3153 47 2828 -22 \n","Q 2503 -91 2169 -91 \n","Q 1331 -91 842 396 \n","Q 353 884 353 1716 \n","Q 353 2575 817 3079 \n","Q 1281 3584 2069 3584 \n","Q 2775 3584 3186 3129 \n","Q 3597 2675 3597 1894 \n","z\n","M 3022 2063 \n","Q 3016 2534 2758 2815 \n","Q 2500 3097 2075 3097 \n","Q 1594 3097 1305 2825 \n","Q 1016 2553 972 2059 \n","L 3022 2063 \n","z\n","\" transform=\"scale(0.015625)\"/>\n","       <path id=\"DejaVuSans-70\" d=\"M 1159 525 \n","L 1159 -1331 \n","L 581 -1331 \n","L 581 3500 \n","L 1159 3500 \n","L 1159 2969 \n","Q 1341 3281 1617 3432 \n","Q 1894 3584 2278 3584 \n","Q 2916 3584 3314 3078 \n","Q 3713 2572 3713 1747 \n","Q 3713 922 3314 415 \n","Q 2916 -91 2278 -91 \n","Q 1894 -91 1617 61 \n","Q 1341 213 1159 525 \n","z\n","M 3116 1747 \n","Q 3116 2381 2855 2742 \n","Q 2594 3103 2138 3103 \n","Q 1681 3103 1420 2742 \n","Q 1159 2381 1159 1747 \n","Q 1159 1113 1420 752 \n","Q 1681 391 2138 391 \n","Q 2594 391 2855 752 \n","Q 3116 1113 3116 1747 \n","z\n","\" transform=\"scale(0.015625)\"/>\n","       <path id=\"DejaVuSans-6f\" d=\"M 1959 3097 \n","Q 1497 3097 1228 2736 \n","Q 959 2375 959 1747 \n","Q 959 1119 1226 758 \n","Q 1494 397 1959 397 \n","Q 2419 397 2687 759 \n","Q 2956 1122 2956 1747 \n","Q 2956 2369 2687 2733 \n","Q 2419 3097 1959 3097 \n","z\n","M 1959 3584 \n","Q 2709 3584 3137 3096 \n","Q 3566 2609 3566 1747 \n","Q 3566 888 3137 398 \n","Q 2709 -91 1959 -91 \n","Q 1206 -91 779 398 \n","Q 353 888 353 1747 \n","Q 353 2609 779 3096 \n","Q 1206 3584 1959 3584 \n","z\n","\" transform=\"scale(0.015625)\"/>\n","       <path id=\"DejaVuSans-63\" d=\"M 3122 3366 \n","L 3122 2828 \n","Q 2878 2963 2633 3030 \n","Q 2388 3097 2138 3097 \n","Q 1578 3097 1268 2742 \n","Q 959 2388 959 1747 \n","Q 959 1106 1268 751 \n","Q 1578 397 2138 397 \n","Q 2388 397 2633 464 \n","Q 2878 531 3122 666 \n","L 3122 134 \n","Q 2881 22 2623 -34 \n","Q 2366 -91 2075 -91 \n","Q 1284 -91 818 406 \n","Q 353 903 353 1747 \n","Q 353 2603 823 3093 \n","Q 1294 3584 2113 3584 \n","Q 2378 3584 2631 3529 \n","Q 2884 3475 3122 3366 \n","z\n","\" transform=\"scale(0.015625)\"/>\n","       <path id=\"DejaVuSans-68\" d=\"M 3513 2113 \n","L 3513 0 \n","L 2938 0 \n","L 2938 2094 \n","Q 2938 2591 2744 2837 \n","Q 2550 3084 2163 3084 \n","Q 1697 3084 1428 2787 \n","Q 1159 2491 1159 1978 \n","L 1159 0 \n","L 581 0 \n","L 581 4863 \n","L 1159 4863 \n","L 1159 2956 \n","Q 1366 3272 1645 3428 \n","Q 1925 3584 2291 3584 \n","Q 2894 3584 3203 3211 \n","Q 3513 2838 3513 2113 \n","z\n","\" transform=\"scale(0.015625)\"/>\n","      </defs>\n","      <use xlink:href=\"#DejaVuSans-65\"/>\n","      <use xlink:href=\"#DejaVuSans-70\" x=\"61.523438\"/>\n","      <use xlink:href=\"#DejaVuSans-6f\" x=\"125\"/>\n","      <use xlink:href=\"#DejaVuSans-63\" x=\"186.181641\"/>\n","      <use xlink:href=\"#DejaVuSans-68\" x=\"241.162109\"/>\n","     </g>\n","    </g>\n","   </g>\n","   <g id=\"matplotlib.axis_2\">\n","    <g id=\"ytick_1\">\n","     <g id=\"line2d_11\">\n","      <path d=\"M 30.103125 115.722932 \n","L 225.403125 115.722932 \n","\" clip-path=\"url(#p4793444f25)\" style=\"fill: none; stroke: #b0b0b0; stroke-width: 0.8; stroke-linecap: square\"/>\n","     </g>\n","     <g id=\"line2d_12\">\n","      <defs>\n","       <path id=\"m6126f38bc5\" d=\"M 0 0 \n","L -3.5 0 \n","\" style=\"stroke: #000000; stroke-width: 0.8\"/>\n","      </defs>\n","      <g>\n","       <use xlink:href=\"#m6126f38bc5\" x=\"30.103125\" y=\"115.722932\" style=\"stroke: #000000; stroke-width: 0.8\"/>\n","      </g>\n","     </g>\n","     <g id=\"text_7\">\n","      <!-- 0.5 -->\n","      <g transform=\"translate(7.2 119.522151)scale(0.1 -0.1)\">\n","       <defs>\n","        <path id=\"DejaVuSans-2e\" d=\"M 684 794 \n","L 1344 794 \n","L 1344 0 \n","L 684 0 \n","L 684 794 \n","z\n","\" transform=\"scale(0.015625)\"/>\n","        <path id=\"DejaVuSans-35\" d=\"M 691 4666 \n","L 3169 4666 \n","L 3169 4134 \n","L 1269 4134 \n","L 1269 2991 \n","Q 1406 3038 1543 3061 \n","Q 1681 3084 1819 3084 \n","Q 2600 3084 3056 2656 \n","Q 3513 2228 3513 1497 \n","Q 3513 744 3044 326 \n","Q 2575 -91 1722 -91 \n","Q 1428 -91 1123 -41 \n","Q 819 9 494 109 \n","L 494 744 \n","Q 775 591 1075 516 \n","Q 1375 441 1709 441 \n","Q 2250 441 2565 725 \n","Q 2881 1009 2881 1497 \n","Q 2881 1984 2565 2268 \n","Q 2250 2553 1709 2553 \n","Q 1456 2553 1204 2497 \n","Q 953 2441 691 2322 \n","L 691 4666 \n","z\n","\" transform=\"scale(0.015625)\"/>\n","       </defs>\n","       <use xlink:href=\"#DejaVuSans-30\"/>\n","       <use xlink:href=\"#DejaVuSans-2e\" x=\"63.623047\"/>\n","       <use xlink:href=\"#DejaVuSans-35\" x=\"95.410156\"/>\n","      </g>\n","     </g>\n","    </g>\n","    <g id=\"ytick_2\">\n","     <g id=\"line2d_13\">\n","      <path d=\"M 30.103125 83.125465 \n","L 225.403125 83.125465 \n","\" clip-path=\"url(#p4793444f25)\" style=\"fill: none; stroke: #b0b0b0; stroke-width: 0.8; stroke-linecap: square\"/>\n","     </g>\n","     <g id=\"line2d_14\">\n","      <g>\n","       <use xlink:href=\"#m6126f38bc5\" x=\"30.103125\" y=\"83.125465\" style=\"stroke: #000000; stroke-width: 0.8\"/>\n","      </g>\n","     </g>\n","     <g id=\"text_8\">\n","      <!-- 1.0 -->\n","      <g transform=\"translate(7.2 86.924684)scale(0.1 -0.1)\">\n","       <use xlink:href=\"#DejaVuSans-31\"/>\n","       <use xlink:href=\"#DejaVuSans-2e\" x=\"63.623047\"/>\n","       <use xlink:href=\"#DejaVuSans-30\" x=\"95.410156\"/>\n","      </g>\n","     </g>\n","    </g>\n","    <g id=\"ytick_3\">\n","     <g id=\"line2d_15\">\n","      <path d=\"M 30.103125 50.527999 \n","L 225.403125 50.527999 \n","\" clip-path=\"url(#p4793444f25)\" style=\"fill: none; stroke: #b0b0b0; stroke-width: 0.8; stroke-linecap: square\"/>\n","     </g>\n","     <g id=\"line2d_16\">\n","      <g>\n","       <use xlink:href=\"#m6126f38bc5\" x=\"30.103125\" y=\"50.527999\" style=\"stroke: #000000; stroke-width: 0.8\"/>\n","      </g>\n","     </g>\n","     <g id=\"text_9\">\n","      <!-- 1.5 -->\n","      <g transform=\"translate(7.2 54.327218)scale(0.1 -0.1)\">\n","       <use xlink:href=\"#DejaVuSans-31\"/>\n","       <use xlink:href=\"#DejaVuSans-2e\" x=\"63.623047\"/>\n","       <use xlink:href=\"#DejaVuSans-35\" x=\"95.410156\"/>\n","      </g>\n","     </g>\n","    </g>\n","    <g id=\"ytick_4\">\n","     <g id=\"line2d_17\">\n","      <path d=\"M 30.103125 17.930532 \n","L 225.403125 17.930532 \n","\" clip-path=\"url(#p4793444f25)\" style=\"fill: none; stroke: #b0b0b0; stroke-width: 0.8; stroke-linecap: square\"/>\n","     </g>\n","     <g id=\"line2d_18\">\n","      <g>\n","       <use xlink:href=\"#m6126f38bc5\" x=\"30.103125\" y=\"17.930532\" style=\"stroke: #000000; stroke-width: 0.8\"/>\n","      </g>\n","     </g>\n","     <g id=\"text_10\">\n","      <!-- 2.0 -->\n","      <g transform=\"translate(7.2 21.729751)scale(0.1 -0.1)\">\n","       <use xlink:href=\"#DejaVuSans-32\"/>\n","       <use xlink:href=\"#DejaVuSans-2e\" x=\"63.623047\"/>\n","       <use xlink:href=\"#DejaVuSans-30\" x=\"95.410156\"/>\n","      </g>\n","     </g>\n","    </g>\n","   </g>\n","   <g id=\"line2d_19\">\n","    <path d=\"M 12.70611 13.377273 \n","L 17.009095 51.38958 \n","L 21.31208 70.153824 \n","L 25.615065 81.044691 \n","L 29.91805 88.351009 \n","L 30.103125 88.577645 \n","L 34.40611 119.422817 \n","L 38.709095 120.03634 \n","L 43.01208 120.893958 \n","L 47.315065 121.715646 \n","L 51.61805 122.20989 \n","L 51.803125 122.222076 \n","L 56.10611 126.015784 \n","L 60.409095 126.242035 \n","L 64.71208 126.422501 \n","L 69.015065 126.70892 \n","L 73.31805 127.124246 \n","L 73.503125 127.163657 \n","L 77.80611 129.09289 \n","L 82.109095 129.13331 \n","L 86.41208 129.371312 \n","L 90.715065 129.583366 \n","L 95.01805 129.682633 \n","L 95.203125 129.679451 \n","L 99.50611 130.815335 \n","L 103.809095 130.85036 \n","L 108.11208 130.915573 \n","L 112.415065 131.228695 \n","L 116.71805 131.342969 \n","L 116.903125 131.347353 \n","L 121.20611 132.856872 \n","L 125.509095 132.835804 \n","L 129.81208 132.812865 \n","L 134.115065 132.694479 \n","L 138.41805 132.755114 \n","L 138.603125 132.760519 \n","L 142.90611 134.458065 \n","L 147.209095 134.217483 \n","L 151.51208 134.005808 \n","L 155.815065 134.036627 \n","L 160.11805 133.933286 \n","L 160.303125 133.94652 \n","L 164.60611 135.496202 \n","L 168.909095 135.093965 \n","L 173.21208 135.066756 \n","L 177.515065 135.012995 \n","L 181.81805 134.919646 \n","L 182.003125 134.906972 \n","L 186.30611 135.446554 \n","L 190.609095 135.665643 \n","L 194.91208 135.902979 \n","L 199.215065 135.887958 \n","L 203.51805 135.773476 \n","L 203.703125 135.781639 \n","L 208.00611 136.871368 \n","L 212.309095 136.85878 \n","L 216.61208 136.922727 \n","L 220.915065 136.708668 \n","L 225.21805 136.782326 \n","L 225.403125 136.774856 \n","\" clip-path=\"url(#p4793444f25)\" style=\"fill: none; stroke: #1f77b4; stroke-width: 1.5; stroke-linecap: square\"/>\n","   </g>\n","   <g id=\"line2d_20\">\n","    <path d=\"M 12.70611 131.698538 \n","L 17.009095 118.480463 \n","L 21.31208 111.792469 \n","L 25.615065 107.883 \n","L 29.91805 105.178046 \n","L 30.103125 105.097244 \n","L 34.40611 93.974857 \n","L 38.709095 93.564103 \n","L 43.01208 93.244627 \n","L 47.315065 92.965771 \n","L 51.61805 92.764501 \n","L 51.803125 92.76019 \n","L 56.10611 91.346029 \n","L 60.409095 91.220065 \n","L 64.71208 91.108705 \n","L 69.015065 91.003734 \n","L 73.31805 90.855315 \n","L 73.503125 90.842372 \n","L 77.80611 90.343789 \n","L 82.109095 90.146627 \n","L 86.41208 90.071778 \n","L 90.715065 89.994648 \n","L 95.01805 89.993278 \n","L 95.203125 89.993752 \n","L 99.50611 89.582524 \n","L 103.809095 89.552402 \n","L 108.11208 89.531408 \n","L 112.415065 89.382624 \n","L 116.71805 89.333881 \n","L 116.903125 89.328763 \n","L 121.20611 88.843166 \n","L 125.509095 88.826736 \n","L 129.81208 88.888806 \n","L 134.115065 88.932163 \n","L 138.41805 88.877122 \n","L 138.603125 88.880005 \n","L 142.90611 88.136669 \n","L 147.209095 88.254418 \n","L 151.51208 88.332005 \n","L 155.815065 88.328354 \n","L 160.11805 88.396265 \n","L 160.303125 88.391043 \n","L 164.60611 88.016181 \n","L 168.909095 88.109285 \n","L 173.21208 88.087378 \n","L 177.515065 88.049041 \n","L 181.81805 88.063281 \n","L 182.003125 88.065068 \n","L 186.30611 87.824495 \n","L 190.609095 87.775205 \n","L 194.91208 87.714961 \n","L 199.215065 87.758775 \n","L 203.51805 87.805874 \n","L 203.703125 87.799942 \n","L 208.00611 87.413741 \n","L 212.309095 87.317898 \n","L 216.61208 87.304206 \n","L 220.915065 87.416479 \n","L 225.21805 87.394025 \n","L 225.403125 87.398993 \n","\" clip-path=\"url(#p4793444f25)\" style=\"fill: none; stroke-dasharray: 5.55,2.4; stroke-dashoffset: 0; stroke: #bf00bf; stroke-width: 1.5\"/>\n","   </g>\n","   <g id=\"line2d_21\">\n","    <path d=\"M 30.103125 95.447308 \n","L 51.803125 91.248754 \n","L 73.503125 92.050652 \n","L 95.203125 90.212155 \n","L 116.903125 89.736232 \n","L 138.603125 89.325504 \n","L 160.303125 88.790905 \n","L 182.003125 88.634437 \n","L 203.703125 88.517086 \n","L 225.403125 88.921295 \n","\" clip-path=\"url(#p4793444f25)\" style=\"fill: none; stroke-dasharray: 9.6,2.4,1.5,2.4; stroke-dashoffset: 0; stroke: #008000; stroke-width: 1.5\"/>\n","   </g>\n","   <g id=\"patch_3\">\n","    <path d=\"M 30.103125 143.1 \n","L 30.103125 7.2 \n","\" style=\"fill: none; stroke: #000000; stroke-width: 0.8; stroke-linejoin: miter; stroke-linecap: square\"/>\n","   </g>\n","   <g id=\"patch_4\">\n","    <path d=\"M 225.403125 143.1 \n","L 225.403125 7.2 \n","\" style=\"fill: none; stroke: #000000; stroke-width: 0.8; stroke-linejoin: miter; stroke-linecap: square\"/>\n","   </g>\n","   <g id=\"patch_5\">\n","    <path d=\"M 30.103125 143.1 \n","L 225.403125 143.1 \n","\" style=\"fill: none; stroke: #000000; stroke-width: 0.8; stroke-linejoin: miter; stroke-linecap: square\"/>\n","   </g>\n","   <g id=\"patch_6\">\n","    <path d=\"M 30.103125 7.2 \n","L 225.403125 7.2 \n","\" style=\"fill: none; stroke: #000000; stroke-width: 0.8; stroke-linejoin: miter; stroke-linecap: square\"/>\n","   </g>\n","   <g id=\"legend_1\">\n","    <g id=\"patch_7\">\n","     <path d=\"M 140.634375 59.234375 \n","L 218.403125 59.234375 \n","Q 220.403125 59.234375 220.403125 57.234375 \n","L 220.403125 14.2 \n","Q 220.403125 12.2 218.403125 12.2 \n","L 140.634375 12.2 \n","Q 138.634375 12.2 138.634375 14.2 \n","L 138.634375 57.234375 \n","Q 138.634375 59.234375 140.634375 59.234375 \n","z\n","\" style=\"fill: #ffffff; opacity: 0.8; stroke: #cccccc; stroke-linejoin: miter\"/>\n","    </g>\n","    <g id=\"line2d_22\">\n","     <path d=\"M 142.634375 20.298438 \n","L 152.634375 20.298438 \n","L 162.634375 20.298438 \n","\" style=\"fill: none; stroke: #1f77b4; stroke-width: 1.5; stroke-linecap: square\"/>\n","    </g>\n","    <g id=\"text_11\">\n","     <!-- train loss -->\n","     <g transform=\"translate(170.634375 23.798438)scale(0.1 -0.1)\">\n","      <defs>\n","       <path id=\"DejaVuSans-74\" d=\"M 1172 4494 \n","L 1172 3500 \n","L 2356 3500 \n","L 2356 3053 \n","L 1172 3053 \n","L 1172 1153 \n","Q 1172 725 1289 603 \n","Q 1406 481 1766 481 \n","L 2356 481 \n","L 2356 0 \n","L 1766 0 \n","Q 1100 0 847 248 \n","Q 594 497 594 1153 \n","L 594 3053 \n","L 172 3053 \n","L 172 3500 \n","L 594 3500 \n","L 594 4494 \n","L 1172 4494 \n","z\n","\" transform=\"scale(0.015625)\"/>\n","       <path id=\"DejaVuSans-72\" d=\"M 2631 2963 \n","Q 2534 3019 2420 3045 \n","Q 2306 3072 2169 3072 \n","Q 1681 3072 1420 2755 \n","Q 1159 2438 1159 1844 \n","L 1159 0 \n","L 581 0 \n","L 581 3500 \n","L 1159 3500 \n","L 1159 2956 \n","Q 1341 3275 1631 3429 \n","Q 1922 3584 2338 3584 \n","Q 2397 3584 2469 3576 \n","Q 2541 3569 2628 3553 \n","L 2631 2963 \n","z\n","\" transform=\"scale(0.015625)\"/>\n","       <path id=\"DejaVuSans-61\" d=\"M 2194 1759 \n","Q 1497 1759 1228 1600 \n","Q 959 1441 959 1056 \n","Q 959 750 1161 570 \n","Q 1363 391 1709 391 \n","Q 2188 391 2477 730 \n","Q 2766 1069 2766 1631 \n","L 2766 1759 \n","L 2194 1759 \n","z\n","M 3341 1997 \n","L 3341 0 \n","L 2766 0 \n","L 2766 531 \n","Q 2569 213 2275 61 \n","Q 1981 -91 1556 -91 \n","Q 1019 -91 701 211 \n","Q 384 513 384 1019 \n","Q 384 1609 779 1909 \n","Q 1175 2209 1959 2209 \n","L 2766 2209 \n","L 2766 2266 \n","Q 2766 2663 2505 2880 \n","Q 2244 3097 1772 3097 \n","Q 1472 3097 1187 3025 \n","Q 903 2953 641 2809 \n","L 641 3341 \n","Q 956 3463 1253 3523 \n","Q 1550 3584 1831 3584 \n","Q 2591 3584 2966 3190 \n","Q 3341 2797 3341 1997 \n","z\n","\" transform=\"scale(0.015625)\"/>\n","       <path id=\"DejaVuSans-69\" d=\"M 603 3500 \n","L 1178 3500 \n","L 1178 0 \n","L 603 0 \n","L 603 3500 \n","z\n","M 603 4863 \n","L 1178 4863 \n","L 1178 4134 \n","L 603 4134 \n","L 603 4863 \n","z\n","\" transform=\"scale(0.015625)\"/>\n","       <path id=\"DejaVuSans-6e\" d=\"M 3513 2113 \n","L 3513 0 \n","L 2938 0 \n","L 2938 2094 \n","Q 2938 2591 2744 2837 \n","Q 2550 3084 2163 3084 \n","Q 1697 3084 1428 2787 \n","Q 1159 2491 1159 1978 \n","L 1159 0 \n","L 581 0 \n","L 581 3500 \n","L 1159 3500 \n","L 1159 2956 \n","Q 1366 3272 1645 3428 \n","Q 1925 3584 2291 3584 \n","Q 2894 3584 3203 3211 \n","Q 3513 2838 3513 2113 \n","z\n","\" transform=\"scale(0.015625)\"/>\n","       <path id=\"DejaVuSans-20\" transform=\"scale(0.015625)\"/>\n","       <path id=\"DejaVuSans-6c\" d=\"M 603 4863 \n","L 1178 4863 \n","L 1178 0 \n","L 603 0 \n","L 603 4863 \n","z\n","\" transform=\"scale(0.015625)\"/>\n","       <path id=\"DejaVuSans-73\" d=\"M 2834 3397 \n","L 2834 2853 \n","Q 2591 2978 2328 3040 \n","Q 2066 3103 1784 3103 \n","Q 1356 3103 1142 2972 \n","Q 928 2841 928 2578 \n","Q 928 2378 1081 2264 \n","Q 1234 2150 1697 2047 \n","L 1894 2003 \n","Q 2506 1872 2764 1633 \n","Q 3022 1394 3022 966 \n","Q 3022 478 2636 193 \n","Q 2250 -91 1575 -91 \n","Q 1294 -91 989 -36 \n","Q 684 19 347 128 \n","L 347 722 \n","Q 666 556 975 473 \n","Q 1284 391 1588 391 \n","Q 1994 391 2212 530 \n","Q 2431 669 2431 922 \n","Q 2431 1156 2273 1281 \n","Q 2116 1406 1581 1522 \n","L 1381 1569 \n","Q 847 1681 609 1914 \n","Q 372 2147 372 2553 \n","Q 372 3047 722 3315 \n","Q 1072 3584 1716 3584 \n","Q 2034 3584 2315 3537 \n","Q 2597 3491 2834 3397 \n","z\n","\" transform=\"scale(0.015625)\"/>\n","      </defs>\n","      <use xlink:href=\"#DejaVuSans-74\"/>\n","      <use xlink:href=\"#DejaVuSans-72\" x=\"39.208984\"/>\n","      <use xlink:href=\"#DejaVuSans-61\" x=\"80.322266\"/>\n","      <use xlink:href=\"#DejaVuSans-69\" x=\"141.601562\"/>\n","      <use xlink:href=\"#DejaVuSans-6e\" x=\"169.384766\"/>\n","      <use xlink:href=\"#DejaVuSans-20\" x=\"232.763672\"/>\n","      <use xlink:href=\"#DejaVuSans-6c\" x=\"264.550781\"/>\n","      <use xlink:href=\"#DejaVuSans-6f\" x=\"292.333984\"/>\n","      <use xlink:href=\"#DejaVuSans-73\" x=\"353.515625\"/>\n","      <use xlink:href=\"#DejaVuSans-73\" x=\"405.615234\"/>\n","     </g>\n","    </g>\n","    <g id=\"line2d_23\">\n","     <path d=\"M 142.634375 34.976562 \n","L 152.634375 34.976562 \n","L 162.634375 34.976562 \n","\" style=\"fill: none; stroke-dasharray: 5.55,2.4; stroke-dashoffset: 0; stroke: #bf00bf; stroke-width: 1.5\"/>\n","    </g>\n","    <g id=\"text_12\">\n","     <!-- train acc -->\n","     <g transform=\"translate(170.634375 38.476562)scale(0.1 -0.1)\">\n","      <use xlink:href=\"#DejaVuSans-74\"/>\n","      <use xlink:href=\"#DejaVuSans-72\" x=\"39.208984\"/>\n","      <use xlink:href=\"#DejaVuSans-61\" x=\"80.322266\"/>\n","      <use xlink:href=\"#DejaVuSans-69\" x=\"141.601562\"/>\n","      <use xlink:href=\"#DejaVuSans-6e\" x=\"169.384766\"/>\n","      <use xlink:href=\"#DejaVuSans-20\" x=\"232.763672\"/>\n","      <use xlink:href=\"#DejaVuSans-61\" x=\"264.550781\"/>\n","      <use xlink:href=\"#DejaVuSans-63\" x=\"325.830078\"/>\n","      <use xlink:href=\"#DejaVuSans-63\" x=\"380.810547\"/>\n","     </g>\n","    </g>\n","    <g id=\"line2d_24\">\n","     <path d=\"M 142.634375 49.654688 \n","L 152.634375 49.654688 \n","L 162.634375 49.654688 \n","\" style=\"fill: none; stroke-dasharray: 9.6,2.4,1.5,2.4; stroke-dashoffset: 0; stroke: #008000; stroke-width: 1.5\"/>\n","    </g>\n","    <g id=\"text_13\">\n","     <!-- test acc -->\n","     <g transform=\"translate(170.634375 53.154688)scale(0.1 -0.1)\">\n","      <use xlink:href=\"#DejaVuSans-74\"/>\n","      <use xlink:href=\"#DejaVuSans-65\" x=\"39.208984\"/>\n","      <use xlink:href=\"#DejaVuSans-73\" x=\"100.732422\"/>\n","      <use xlink:href=\"#DejaVuSans-74\" x=\"152.832031\"/>\n","      <use xlink:href=\"#DejaVuSans-20\" x=\"192.041016\"/>\n","      <use xlink:href=\"#DejaVuSans-61\" x=\"223.828125\"/>\n","      <use xlink:href=\"#DejaVuSans-63\" x=\"285.107422\"/>\n","      <use xlink:href=\"#DejaVuSans-63\" x=\"340.087891\"/>\n","     </g>\n","    </g>\n","   </g>\n","  </g>\n"," </g>\n"," <defs>\n","  <clipPath id=\"p4793444f25\">\n","   <rect x=\"30.103125\" y=\"7.2\" width=\"195.3\" height=\"135.9\"/>\n","  </clipPath>\n"," </defs>\n","</svg>\n"],"text/plain":["<Figure size 252x180 with 1 Axes>"]},"metadata":{"needs_background":"light"},"output_type":"display_data"}],"source":["lr, num_epochs, batch_size = 0.05, 10, 128\n","train_iter, test_iter = d2l.load_data_fashion_mnist(batch_size, resize=224)\n","d2l.train_ch6(net, train_iter, test_iter, num_epochs, lr, d2l.try_gpu())"],"id":"c6a88053"},{"cell_type":"markdown","metadata":{"origin_pos":21,"id":"ddc15869"},"source":["## 小结\n","\n","* VGG-11使用可复用的卷积块构造网络。不同的VGG模型可通过每个块中卷积层数量和输出通道数量的差异来定义。\n","* 块的使用导致网络定义的非常简洁。使用块可以有效地设计复杂的网络。\n","* 在VGG论文中，Simonyan和Ziserman尝试了各种架构。特别是他们发现深层且窄的卷积（即$3 \\times 3$）比较浅层且宽的卷积更有效。\n"],"id":"ddc15869"},{"cell_type":"markdown","metadata":{"origin_pos":0,"id":"d454d765"},"source":["# 批量规范化\n",":label:`sec_batch_norm`\n","\n","训练深层神经网络是十分困难的，特别是在较短的时间内使他们收敛更加棘手。\n","在本节中，我们将介绍*批量规范化*（batch normalization） :cite:`Ioffe.Szegedy.2015`，这是一种流行且有效的技术，可持续加速深层网络的收敛速度。\n","再结合在 :numref:`sec_resnet`中将介绍的残差块，批量规范化使得研究人员能够训练100层以上的网络。\n","\n","## 训练深层网络\n","\n","为什么需要批量规范化层呢？让我们来回顾一下训练神经网络时出现的一些实际挑战。\n","\n","首先，数据预处理的方式通常会对最终结果产生巨大影响。\n","回想一下我们应用多层感知机来预测房价的例子（ :numref:`sec_kaggle_house`）。\n","使用真实数据时，我们的第一步是标准化输入特征，使其平均值为0，方差为1。\n","直观地说，这种标准化可以很好地与我们的优化器配合使用，因为它可以将参数的量级进行统一。\n","\n","第二，对于典型的多层感知机或卷积神经网络。当我们训练时，中间层中的变量（例如，多层感知机中的仿射变换输出）可能具有更广的变化范围：不论是沿着从输入到输出的层，跨同一层中的单元，或是随着时间的推移，模型参数的随着训练更新变幻莫测。\n","批量规范化的发明者非正式地假设，这些变量分布中的这种偏移可能会阻碍网络的收敛。\n","直观地说，我们可能会猜想，如果一个层的可变值是另一层的100倍，这可能需要对学习率进行补偿调整。\n","\n","第三，更深层的网络很复杂，容易过拟合。\n","这意味着正则化变得更加重要。\n","\n","批量规范化应用于单个可选层（也可以应用到所有层），其原理如下：在每次训练迭代中，我们首先规范化输入，即通过减去其均值并除以其标准差，其中两者均基于当前小批量处理。\n","接下来，我们应用比例系数和比例偏移。\n","正是由于这个基于*批量*统计的*标准化*，才有了*批量规范化*的名称。\n","\n","请注意，如果我们尝试使用大小为1的小批量应用批量规范化，我们将无法学到任何东西。\n","这是因为在减去均值之后，每个隐藏单元将为0。\n","所以，只有使用足够大的小批量，批量规范化这种方法才是有效且稳定的。\n","请注意，在应用批量规范化时，批量大小的选择可能比没有批量规范化时更重要。\n","\n","从形式上来说，用$\\mathbf{x} \\in \\mathcal{B}$表示一个来自小批量$\\mathcal{B}$的输入，批量规范化$\\mathrm{BN}$根据以下表达式转换$\\mathbf{x}$：\n","\n","$$\\mathrm{BN}(\\mathbf{x}) = \\boldsymbol{\\gamma} \\odot \\frac{\\mathbf{x} - \\hat{\\boldsymbol{\\mu}}_\\mathcal{B}}{\\hat{\\boldsymbol{\\sigma}}_\\mathcal{B}} + \\boldsymbol{\\beta}.$$\n",":eqlabel:`eq_batchnorm`\n","\n","在 :eqref:`eq_batchnorm`中，$\\hat{\\boldsymbol{\\mu}}_\\mathcal{B}$是小批量$\\mathcal{B}$的样本均值，$\\hat{\\boldsymbol{\\sigma}}_\\mathcal{B}$是小批量$\\mathcal{B}$的样本标准差。\n","应用标准化后，生成的小批量的平均值为0和单位方差为1。\n","由于单位方差（与其他一些魔法数）是一个主观的选择，因此我们通常包含\n","*拉伸参数*（scale）$\\boldsymbol{\\gamma}$和*偏移参数*（shift）$\\boldsymbol{\\beta}$，它们的形状与$\\mathbf{x}$相同。\n","请注意，$\\boldsymbol{\\gamma}$和$\\boldsymbol{\\beta}$是需要与其他模型参数一起学习的参数。\n","\n","由于在训练过程中，中间层的变化幅度不能过于剧烈，而批量规范化将每一层主动居中，并将它们重新调整为给定的平均值和大小（通过$\\hat{\\boldsymbol{\\mu}}_\\mathcal{B}$和${\\hat{\\boldsymbol{\\sigma}}_\\mathcal{B}}$）。\n","\n","从形式上来看，我们计算出 :eqref:`eq_batchnorm`中的$\\hat{\\boldsymbol{\\mu}}_\\mathcal{B}$和${\\hat{\\boldsymbol{\\sigma}}_\\mathcal{B}}$，如下所示：\n","\n","$$\\begin{aligned} \\hat{\\boldsymbol{\\mu}}_\\mathcal{B} &= \\frac{1}{|\\mathcal{B}|} \\sum_{\\mathbf{x} \\in \\mathcal{B}} \\mathbf{x},\\\\\n","\\hat{\\boldsymbol{\\sigma}}_\\mathcal{B}^2 &= \\frac{1}{|\\mathcal{B}|} \\sum_{\\mathbf{x} \\in \\mathcal{B}} (\\mathbf{x} - \\hat{\\boldsymbol{\\mu}}_{\\mathcal{B}})^2 + \\epsilon.\\end{aligned}$$\n","\n","请注意，我们在方差估计值中添加一个小的常量$\\epsilon > 0$，以确保我们永远不会尝试除以零，即使在经验方差估计值可能消失的情况下也是如此。估计值$\\hat{\\boldsymbol{\\mu}}_\\mathcal{B}$和${\\hat{\\boldsymbol{\\sigma}}_\\mathcal{B}}$通过使用平均值和方差的噪声（noise）估计来抵消缩放问题。\n","你可能会认为这种噪声是一个问题，而事实上它是有益的。\n","\n","事实证明，这是深度学习中一个反复出现的主题。\n","由于尚未在理论上明确的原因，优化中的各种噪声源通常会导致更快的训练和较少的过拟合：这种变化似乎是正则化的一种形式。\n","在一些初步研究中， :cite:`Teye.Azizpour.Smith.2018`和 :cite:`Luo.Wang.Shao.ea.2018`分别将批量规范化的性质与贝叶斯先验相关联。\n","这些理论揭示了为什么批量规范化最适应$50 \\sim 100$范围中的中等批量大小的难题。\n","\n","另外，批量规范化层在”训练模式“（通过小批量统计数据规范化）和“预测模式”（通过数据集统计规范化）中的功能不同。\n","在训练过程中，我们无法得知使用整个数据集来估计平均值和方差，所以只能根据每个小批次的平均值和方差不断训练模型。\n","而在预测模式下，可以根据整个数据集精确计算批量规范化所需的平均值和方差。\n","\n","现在，我们了解一下批量规范化在实践中是如何工作的。\n","\n","## 批量规范化层\n","\n","回想一下，批量规范化和其他层之间的一个关键区别是，由于批量规范化在完整的小批量上运行，因此我们不能像以前在引入其他层时那样忽略批量大小。\n","我们在下面讨论这两种情况：全连接层和卷积层，他们的批量规范化实现略有不同。\n","\n","### 全连接层\n","\n","通常，我们将批量规范化层置于全连接层中的仿射变换和激活函数之间。\n","设全连接层的输入为x，权重参数和偏置参数分别为$\\mathbf{W}$和$\\mathbf{b}$，激活函数为$\\phi$，批量规范化的运算符为$\\mathrm{BN}$。\n","那么，使用批量规范化的全连接层的输出的计算详情如下：\n","\n","$$\\mathbf{h} = \\phi(\\mathrm{BN}(\\mathbf{W}\\mathbf{x} + \\mathbf{b}) ).$$\n","\n","回想一下，均值和方差是在应用变换的\"相同\"小批量上计算的。\n","\n","### 卷积层\n","\n","同样，对于卷积层，我们可以在卷积层之后和非线性激活函数之前应用批量规范化。\n","当卷积有多个输出通道时，我们需要对这些通道的“每个”输出执行批量规范化，每个通道都有自己的拉伸（scale）和偏移（shift）参数，这两个参数都是标量。\n","假设我们的小批量包含$m$个样本，并且对于每个通道，卷积的输出具有高度$p$和宽度$q$。\n","那么对于卷积层，我们在每个输出通道的$m \\cdot p \\cdot q$个元素上同时执行每个批量规范化。\n","因此，在计算平均值和方差时，我们会收集所有空间位置的值，然后在给定通道内应用相同的均值和方差，以便在每个空间位置对值进行规范化。\n","\n","### 预测过程中的批量规范化\n","\n","正如我们前面提到的，批量规范化在训练模式和预测模式下的行为通常不同。\n","首先，将训练好的模型用于预测时，我们不再需要样本均值中的噪声以及在微批次上估计每个小批次产生的样本方差了。\n","其次，例如，我们可能需要使用我们的模型对逐个样本进行预测。\n","一种常用的方法是通过移动平均估算整个训练数据集的样本均值和方差，并在预测时使用它们得到确定的输出。\n","可见，和暂退法一样，批量规范化层在训练模式和预测模式下的计算结果也是不一样的。\n","\n","## (**从零实现**)\n","\n","下面，我们从头开始实现一个具有张量的批量规范化层。\n"],"id":"d454d765"},{"cell_type":"code","execution_count":null,"metadata":{"execution":{"iopub.execute_input":"2022-07-31T02:29:50.001576Z","iopub.status.busy":"2022-07-31T02:29:50.000944Z","iopub.status.idle":"2022-07-31T02:29:52.186800Z","shell.execute_reply":"2022-07-31T02:29:52.185838Z"},"origin_pos":2,"tab":["pytorch"],"id":"068b269c"},"outputs":[],"source":["import torch\n","from torch import nn\n","from d2l import torch as d2l\n","\n","\n","def batch_norm(X, gamma, beta, moving_mean, moving_var, eps, momentum):\n","    # 通过is_grad_enabled来判断当前模式是训练模式还是预测模式\n","    if not torch.is_grad_enabled():\n","        # 如果是在预测模式下，直接使用传入的移动平均所得的均值和方差\n","        X_hat = (X - moving_mean) / torch.sqrt(moving_var + eps)\n","    else:\n","        assert len(X.shape) in (2, 4)\n","        if len(X.shape) == 2:\n","            # 使用全连接层的情况，计算特征维上的均值和方差\n","            mean = X.mean(dim=0)\n","            var = ((X - mean) ** 2).mean(dim=0)\n","        else:\n","            # 使用二维卷积层的情况，计算通道维上（axis=1）的均值和方差。\n","            # 这里我们需要保持X的形状以便后面可以做广播运算\n","            mean = X.mean(dim=(0, 2, 3), keepdim=True)\n","            var = ((X - mean) ** 2).mean(dim=(0, 2, 3), keepdim=True)\n","        # 训练模式下，用当前的均值和方差做标准化\n","        X_hat = (X - mean) / torch.sqrt(var + eps)\n","        # 更新移动平均的均值和方差\n","        moving_mean = momentum * moving_mean + (1.0 - momentum) * mean\n","        moving_var = momentum * moving_var + (1.0 - momentum) * var\n","    Y = gamma * X_hat + beta  # 缩放和移位\n","    return Y, moving_mean.data, moving_var.data"],"id":"068b269c"},{"cell_type":"markdown","metadata":{"origin_pos":4,"id":"06e62324"},"source":["我们现在可以[**创建一个正确的`BatchNorm`层**]。\n","这个层将保持适当的参数：拉伸`gamma`和偏移`beta`,这两个参数将在训练过程中更新。\n","此外，我们的层将保存均值和方差的移动平均值，以便在模型预测期间随后使用。\n","\n","撇开算法细节，注意我们实现层的基础设计模式。\n","通常情况下，我们用一个单独的函数定义其数学原理，比如说`batch_norm`。\n","然后，我们将此功能集成到一个自定义层中，其代码主要处理数据移动到训练设备（如GPU）、分配和初始化任何必需的变量、跟踪移动平均线（此处为均值和方差）等问题。\n","为了方便起见，我们并不担心在这里自动推断输入形状，因此我们需要指定整个特征的数量。\n","不用担心，深度学习框架中的批量规范化API将为我们解决上述问题，我们稍后将展示这一点。\n"],"id":"06e62324"},{"cell_type":"code","execution_count":null,"metadata":{"execution":{"iopub.execute_input":"2022-07-31T02:29:52.191211Z","iopub.status.busy":"2022-07-31T02:29:52.190889Z","iopub.status.idle":"2022-07-31T02:29:52.198683Z","shell.execute_reply":"2022-07-31T02:29:52.197807Z"},"origin_pos":6,"tab":["pytorch"],"id":"332e16b8"},"outputs":[],"source":["class BatchNorm(nn.Module):\n","    # num_features：完全连接层的输出数量或卷积层的输出通道数。\n","    # num_dims：2表示完全连接层，4表示卷积层\n","    def __init__(self, num_features, num_dims):\n","        super().__init__()\n","        if num_dims == 2:\n","            shape = (1, num_features)\n","        else:\n","            shape = (1, num_features, 1, 1)\n","        # 参与求梯度和迭代的拉伸和偏移参数，分别初始化成1和0\n","        self.gamma = nn.Parameter(torch.ones(shape))\n","        self.beta = nn.Parameter(torch.zeros(shape))\n","        # 非模型参数的变量初始化为0和1\n","        self.moving_mean = torch.zeros(shape)\n","        self.moving_var = torch.ones(shape)\n","\n","    def forward(self, X):\n","        # 如果X不在内存上，将moving_mean和moving_var\n","        # 复制到X所在显存上\n","        if self.moving_mean.device != X.device:\n","            self.moving_mean = self.moving_mean.to(X.device)\n","            self.moving_var = self.moving_var.to(X.device)\n","        # 保存更新过的moving_mean和moving_var\n","        Y, self.moving_mean, self.moving_var = batch_norm(\n","            X, self.gamma, self.beta, self.moving_mean,\n","            self.moving_var, eps=1e-5, momentum=0.9)\n","        return Y"],"id":"332e16b8"},{"cell_type":"markdown","metadata":{"origin_pos":8,"id":"6b13200d"},"source":["##  使用批量规范化层的 LeNet\n","\n","为了更好理解如何[**应用`BatchNorm`**]，下面我们将其应用(**于LeNet模型**)（ :numref:`sec_lenet`）。\n","回想一下，批量规范化是在卷积层或全连接层之后、相应的激活函数之前应用的。\n"],"id":"6b13200d"},{"cell_type":"code","execution_count":null,"metadata":{"execution":{"iopub.execute_input":"2022-07-31T02:29:52.201762Z","iopub.status.busy":"2022-07-31T02:29:52.201388Z","iopub.status.idle":"2022-07-31T02:29:52.211347Z","shell.execute_reply":"2022-07-31T02:29:52.210339Z"},"origin_pos":10,"tab":["pytorch"],"id":"2eef9b5c"},"outputs":[],"source":["net = nn.Sequential(\n","    nn.Conv2d(1, 6, kernel_size=5), BatchNorm(6, num_dims=4), nn.Sigmoid(),\n","    nn.AvgPool2d(kernel_size=2, stride=2),\n","    nn.Conv2d(6, 16, kernel_size=5), BatchNorm(16, num_dims=4), nn.Sigmoid(),\n","    nn.AvgPool2d(kernel_size=2, stride=2), nn.Flatten(),\n","    nn.Linear(16*4*4, 120), BatchNorm(120, num_dims=2), nn.Sigmoid(),\n","    nn.Linear(120, 84), BatchNorm(84, num_dims=2), nn.Sigmoid(),\n","    nn.Linear(84, 10))"],"id":"2eef9b5c"},{"cell_type":"markdown","metadata":{"origin_pos":12,"id":"c4aff1c1"},"source":["和以前一样，我们将[**在Fashion-MNIST数据集上训练网络**]。\n","这个代码与我们第一次训练LeNet（ :numref:`sec_lenet`）时几乎完全相同，主要区别在于学习率大得多。\n"],"id":"c4aff1c1"},{"cell_type":"code","execution_count":null,"metadata":{"execution":{"iopub.execute_input":"2022-07-31T02:29:52.215820Z","iopub.status.busy":"2022-07-31T02:29:52.215143Z","iopub.status.idle":"2022-07-31T02:30:38.981311Z","shell.execute_reply":"2022-07-31T02:30:38.980591Z"},"origin_pos":13,"tab":["pytorch"],"id":"a19ac3fc","outputId":"30468e72-6397-4b1c-c6b1-8f765cab8e19"},"outputs":[{"name":"stdout","output_type":"stream","text":["loss 0.268, train acc 0.900, test acc 0.831\n","38739.6 examples/sec on cuda:0\n"]},{"data":{"image/svg+xml":["<?xml version=\"1.0\" encoding=\"utf-8\" standalone=\"no\"?>\n","<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\n","  \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\n","<svg xmlns:xlink=\"http://www.w3.org/1999/xlink\" width=\"238.965625pt\" height=\"180.65625pt\" viewBox=\"0 0 238.965625 180.65625\" xmlns=\"http://www.w3.org/2000/svg\" version=\"1.1\">\n"," <metadata>\n","  <rdf:RDF xmlns:dc=\"http://purl.org/dc/elements/1.1/\" xmlns:cc=\"http://creativecommons.org/ns#\" xmlns:rdf=\"http://www.w3.org/1999/02/22-rdf-syntax-ns#\">\n","   <cc:Work>\n","    <dc:type rdf:resource=\"http://purl.org/dc/dcmitype/StillImage\"/>\n","    <dc:date>2022-07-31T02:30:38.937109</dc:date>\n","    <dc:format>image/svg+xml</dc:format>\n","    <dc:creator>\n","     <cc:Agent>\n","      <dc:title>Matplotlib v3.5.1, https://matplotlib.org/</dc:title>\n","     </cc:Agent>\n","    </dc:creator>\n","   </cc:Work>\n","  </rdf:RDF>\n"," </metadata>\n"," <defs>\n","  <style type=\"text/css\">*{stroke-linejoin: round; stroke-linecap: butt}</style>\n"," </defs>\n"," <g id=\"figure_1\">\n","  <g id=\"patch_1\">\n","   <path d=\"M 0 180.65625 \n","L 238.965625 180.65625 \n","L 238.965625 0 \n","L 0 0 \n","L 0 180.65625 \n","z\n","\" style=\"fill: none\"/>\n","  </g>\n","  <g id=\"axes_1\">\n","   <g id=\"patch_2\">\n","    <path d=\"M 30.103125 143.1 \n","L 225.403125 143.1 \n","L 225.403125 7.2 \n","L 30.103125 7.2 \n","z\n","\" style=\"fill: #ffffff\"/>\n","   </g>\n","   <g id=\"matplotlib.axis_1\">\n","    <g id=\"xtick_1\">\n","     <g id=\"line2d_1\">\n","      <path d=\"M 51.803125 143.1 \n","L 51.803125 7.2 \n","\" clip-path=\"url(#p0c662c7060)\" style=\"fill: none; stroke: #b0b0b0; stroke-width: 0.8; stroke-linecap: square\"/>\n","     </g>\n","     <g id=\"line2d_2\">\n","      <defs>\n","       <path id=\"m93c6bd4474\" d=\"M 0 0 \n","L 0 3.5 \n","\" style=\"stroke: #000000; stroke-width: 0.8\"/>\n","      </defs>\n","      <g>\n","       <use xlink:href=\"#m93c6bd4474\" x=\"51.803125\" y=\"143.1\" style=\"stroke: #000000; stroke-width: 0.8\"/>\n","      </g>\n","     </g>\n","     <g id=\"text_1\">\n","      <!-- 2 -->\n","      <g transform=\"translate(48.621875 157.698438)scale(0.1 -0.1)\">\n","       <defs>\n","        <path id=\"DejaVuSans-32\" d=\"M 1228 531 \n","L 3431 531 \n","L 3431 0 \n","L 469 0 \n","L 469 531 \n","Q 828 903 1448 1529 \n","Q 2069 2156 2228 2338 \n","Q 2531 2678 2651 2914 \n","Q 2772 3150 2772 3378 \n","Q 2772 3750 2511 3984 \n","Q 2250 4219 1831 4219 \n","Q 1534 4219 1204 4116 \n","Q 875 4013 500 3803 \n","L 500 4441 \n","Q 881 4594 1212 4672 \n","Q 1544 4750 1819 4750 \n","Q 2544 4750 2975 4387 \n","Q 3406 4025 3406 3419 \n","Q 3406 3131 3298 2873 \n","Q 3191 2616 2906 2266 \n","Q 2828 2175 2409 1742 \n","Q 1991 1309 1228 531 \n","z\n","\" transform=\"scale(0.015625)\"/>\n","       </defs>\n","       <use xlink:href=\"#DejaVuSans-32\"/>\n","      </g>\n","     </g>\n","    </g>\n","    <g id=\"xtick_2\">\n","     <g id=\"line2d_3\">\n","      <path d=\"M 95.203125 143.1 \n","L 95.203125 7.2 \n","\" clip-path=\"url(#p0c662c7060)\" style=\"fill: none; stroke: #b0b0b0; stroke-width: 0.8; stroke-linecap: square\"/>\n","     </g>\n","     <g id=\"line2d_4\">\n","      <g>\n","       <use xlink:href=\"#m93c6bd4474\" x=\"95.203125\" y=\"143.1\" style=\"stroke: #000000; stroke-width: 0.8\"/>\n","      </g>\n","     </g>\n","     <g id=\"text_2\">\n","      <!-- 4 -->\n","      <g transform=\"translate(92.021875 157.698438)scale(0.1 -0.1)\">\n","       <defs>\n","        <path id=\"DejaVuSans-34\" d=\"M 2419 4116 \n","L 825 1625 \n","L 2419 1625 \n","L 2419 4116 \n","z\n","M 2253 4666 \n","L 3047 4666 \n","L 3047 1625 \n","L 3713 1625 \n","L 3713 1100 \n","L 3047 1100 \n","L 3047 0 \n","L 2419 0 \n","L 2419 1100 \n","L 313 1100 \n","L 313 1709 \n","L 2253 4666 \n","z\n","\" transform=\"scale(0.015625)\"/>\n","       </defs>\n","       <use xlink:href=\"#DejaVuSans-34\"/>\n","      </g>\n","     </g>\n","    </g>\n","    <g id=\"xtick_3\">\n","     <g id=\"line2d_5\">\n","      <path d=\"M 138.603125 143.1 \n","L 138.603125 7.2 \n","\" clip-path=\"url(#p0c662c7060)\" style=\"fill: none; stroke: #b0b0b0; stroke-width: 0.8; stroke-linecap: square\"/>\n","     </g>\n","     <g id=\"line2d_6\">\n","      <g>\n","       <use xlink:href=\"#m93c6bd4474\" x=\"138.603125\" y=\"143.1\" style=\"stroke: #000000; stroke-width: 0.8\"/>\n","      </g>\n","     </g>\n","     <g id=\"text_3\">\n","      <!-- 6 -->\n","      <g transform=\"translate(135.421875 157.698438)scale(0.1 -0.1)\">\n","       <defs>\n","        <path id=\"DejaVuSans-36\" d=\"M 2113 2584 \n","Q 1688 2584 1439 2293 \n","Q 1191 2003 1191 1497 \n","Q 1191 994 1439 701 \n","Q 1688 409 2113 409 \n","Q 2538 409 2786 701 \n","Q 3034 994 3034 1497 \n","Q 3034 2003 2786 2293 \n","Q 2538 2584 2113 2584 \n","z\n","M 3366 4563 \n","L 3366 3988 \n","Q 3128 4100 2886 4159 \n","Q 2644 4219 2406 4219 \n","Q 1781 4219 1451 3797 \n","Q 1122 3375 1075 2522 \n","Q 1259 2794 1537 2939 \n","Q 1816 3084 2150 3084 \n","Q 2853 3084 3261 2657 \n","Q 3669 2231 3669 1497 \n","Q 3669 778 3244 343 \n","Q 2819 -91 2113 -91 \n","Q 1303 -91 875 529 \n","Q 447 1150 447 2328 \n","Q 447 3434 972 4092 \n","Q 1497 4750 2381 4750 \n","Q 2619 4750 2861 4703 \n","Q 3103 4656 3366 4563 \n","z\n","\" transform=\"scale(0.015625)\"/>\n","       </defs>\n","       <use xlink:href=\"#DejaVuSans-36\"/>\n","      </g>\n","     </g>\n","    </g>\n","    <g id=\"xtick_4\">\n","     <g id=\"line2d_7\">\n","      <path d=\"M 182.003125 143.1 \n","L 182.003125 7.2 \n","\" clip-path=\"url(#p0c662c7060)\" style=\"fill: none; stroke: #b0b0b0; stroke-width: 0.8; stroke-linecap: square\"/>\n","     </g>\n","     <g id=\"line2d_8\">\n","      <g>\n","       <use xlink:href=\"#m93c6bd4474\" x=\"182.003125\" y=\"143.1\" style=\"stroke: #000000; stroke-width: 0.8\"/>\n","      </g>\n","     </g>\n","     <g id=\"text_4\">\n","      <!-- 8 -->\n","      <g transform=\"translate(178.821875 157.698438)scale(0.1 -0.1)\">\n","       <defs>\n","        <path id=\"DejaVuSans-38\" d=\"M 2034 2216 \n","Q 1584 2216 1326 1975 \n","Q 1069 1734 1069 1313 \n","Q 1069 891 1326 650 \n","Q 1584 409 2034 409 \n","Q 2484 409 2743 651 \n","Q 3003 894 3003 1313 \n","Q 3003 1734 2745 1975 \n","Q 2488 2216 2034 2216 \n","z\n","M 1403 2484 \n","Q 997 2584 770 2862 \n","Q 544 3141 544 3541 \n","Q 544 4100 942 4425 \n","Q 1341 4750 2034 4750 \n","Q 2731 4750 3128 4425 \n","Q 3525 4100 3525 3541 \n","Q 3525 3141 3298 2862 \n","Q 3072 2584 2669 2484 \n","Q 3125 2378 3379 2068 \n","Q 3634 1759 3634 1313 \n","Q 3634 634 3220 271 \n","Q 2806 -91 2034 -91 \n","Q 1263 -91 848 271 \n","Q 434 634 434 1313 \n","Q 434 1759 690 2068 \n","Q 947 2378 1403 2484 \n","z\n","M 1172 3481 \n","Q 1172 3119 1398 2916 \n","Q 1625 2713 2034 2713 \n","Q 2441 2713 2670 2916 \n","Q 2900 3119 2900 3481 \n","Q 2900 3844 2670 4047 \n","Q 2441 4250 2034 4250 \n","Q 1625 4250 1398 4047 \n","Q 1172 3844 1172 3481 \n","z\n","\" transform=\"scale(0.015625)\"/>\n","       </defs>\n","       <use xlink:href=\"#DejaVuSans-38\"/>\n","      </g>\n","     </g>\n","    </g>\n","    <g id=\"xtick_5\">\n","     <g id=\"line2d_9\">\n","      <path d=\"M 225.403125 143.1 \n","L 225.403125 7.2 \n","\" clip-path=\"url(#p0c662c7060)\" style=\"fill: none; stroke: #b0b0b0; stroke-width: 0.8; stroke-linecap: square\"/>\n","     </g>\n","     <g id=\"line2d_10\">\n","      <g>\n","       <use xlink:href=\"#m93c6bd4474\" x=\"225.403125\" y=\"143.1\" style=\"stroke: #000000; stroke-width: 0.8\"/>\n","      </g>\n","     </g>\n","     <g id=\"text_5\">\n","      <!-- 10 -->\n","      <g transform=\"translate(219.040625 157.698438)scale(0.1 -0.1)\">\n","       <defs>\n","        <path id=\"DejaVuSans-31\" d=\"M 794 531 \n","L 1825 531 \n","L 1825 4091 \n","L 703 3866 \n","L 703 4441 \n","L 1819 4666 \n","L 2450 4666 \n","L 2450 531 \n","L 3481 531 \n","L 3481 0 \n","L 794 0 \n","L 794 531 \n","z\n","\" transform=\"scale(0.015625)\"/>\n","        <path id=\"DejaVuSans-30\" d=\"M 2034 4250 \n","Q 1547 4250 1301 3770 \n","Q 1056 3291 1056 2328 \n","Q 1056 1369 1301 889 \n","Q 1547 409 2034 409 \n","Q 2525 409 2770 889 \n","Q 3016 1369 3016 2328 \n","Q 3016 3291 2770 3770 \n","Q 2525 4250 2034 4250 \n","z\n","M 2034 4750 \n","Q 2819 4750 3233 4129 \n","Q 3647 3509 3647 2328 \n","Q 3647 1150 3233 529 \n","Q 2819 -91 2034 -91 \n","Q 1250 -91 836 529 \n","Q 422 1150 422 2328 \n","Q 422 3509 836 4129 \n","Q 1250 4750 2034 4750 \n","z\n","\" transform=\"scale(0.015625)\"/>\n","       </defs>\n","       <use xlink:href=\"#DejaVuSans-31\"/>\n","       <use xlink:href=\"#DejaVuSans-30\" x=\"63.623047\"/>\n","      </g>\n","     </g>\n","    </g>\n","    <g id=\"text_6\">\n","     <!-- epoch -->\n","     <g transform=\"translate(112.525 171.376563)scale(0.1 -0.1)\">\n","      <defs>\n","       <path id=\"DejaVuSans-65\" d=\"M 3597 1894 \n","L 3597 1613 \n","L 953 1613 \n","Q 991 1019 1311 708 \n","Q 1631 397 2203 397 \n","Q 2534 397 2845 478 \n","Q 3156 559 3463 722 \n","L 3463 178 \n","Q 3153 47 2828 -22 \n","Q 2503 -91 2169 -91 \n","Q 1331 -91 842 396 \n","Q 353 884 353 1716 \n","Q 353 2575 817 3079 \n","Q 1281 3584 2069 3584 \n","Q 2775 3584 3186 3129 \n","Q 3597 2675 3597 1894 \n","z\n","M 3022 2063 \n","Q 3016 2534 2758 2815 \n","Q 2500 3097 2075 3097 \n","Q 1594 3097 1305 2825 \n","Q 1016 2553 972 2059 \n","L 3022 2063 \n","z\n","\" transform=\"scale(0.015625)\"/>\n","       <path id=\"DejaVuSans-70\" d=\"M 1159 525 \n","L 1159 -1331 \n","L 581 -1331 \n","L 581 3500 \n","L 1159 3500 \n","L 1159 2969 \n","Q 1341 3281 1617 3432 \n","Q 1894 3584 2278 3584 \n","Q 2916 3584 3314 3078 \n","Q 3713 2572 3713 1747 \n","Q 3713 922 3314 415 \n","Q 2916 -91 2278 -91 \n","Q 1894 -91 1617 61 \n","Q 1341 213 1159 525 \n","z\n","M 3116 1747 \n","Q 3116 2381 2855 2742 \n","Q 2594 3103 2138 3103 \n","Q 1681 3103 1420 2742 \n","Q 1159 2381 1159 1747 \n","Q 1159 1113 1420 752 \n","Q 1681 391 2138 391 \n","Q 2594 391 2855 752 \n","Q 3116 1113 3116 1747 \n","z\n","\" transform=\"scale(0.015625)\"/>\n","       <path id=\"DejaVuSans-6f\" d=\"M 1959 3097 \n","Q 1497 3097 1228 2736 \n","Q 959 2375 959 1747 \n","Q 959 1119 1226 758 \n","Q 1494 397 1959 397 \n","Q 2419 397 2687 759 \n","Q 2956 1122 2956 1747 \n","Q 2956 2369 2687 2733 \n","Q 2419 3097 1959 3097 \n","z\n","M 1959 3584 \n","Q 2709 3584 3137 3096 \n","Q 3566 2609 3566 1747 \n","Q 3566 888 3137 398 \n","Q 2709 -91 1959 -91 \n","Q 1206 -91 779 398 \n","Q 353 888 353 1747 \n","Q 353 2609 779 3096 \n","Q 1206 3584 1959 3584 \n","z\n","\" transform=\"scale(0.015625)\"/>\n","       <path id=\"DejaVuSans-63\" d=\"M 3122 3366 \n","L 3122 2828 \n","Q 2878 2963 2633 3030 \n","Q 2388 3097 2138 3097 \n","Q 1578 3097 1268 2742 \n","Q 959 2388 959 1747 \n","Q 959 1106 1268 751 \n","Q 1578 397 2138 397 \n","Q 2388 397 2633 464 \n","Q 2878 531 3122 666 \n","L 3122 134 \n","Q 2881 22 2623 -34 \n","Q 2366 -91 2075 -91 \n","Q 1284 -91 818 406 \n","Q 353 903 353 1747 \n","Q 353 2603 823 3093 \n","Q 1294 3584 2113 3584 \n","Q 2378 3584 2631 3529 \n","Q 2884 3475 3122 3366 \n","z\n","\" transform=\"scale(0.015625)\"/>\n","       <path id=\"DejaVuSans-68\" d=\"M 3513 2113 \n","L 3513 0 \n","L 2938 0 \n","L 2938 2094 \n","Q 2938 2591 2744 2837 \n","Q 2550 3084 2163 3084 \n","Q 1697 3084 1428 2787 \n","Q 1159 2491 1159 1978 \n","L 1159 0 \n","L 581 0 \n","L 581 4863 \n","L 1159 4863 \n","L 1159 2956 \n","Q 1366 3272 1645 3428 \n","Q 1925 3584 2291 3584 \n","Q 2894 3584 3203 3211 \n","Q 3513 2838 3513 2113 \n","z\n","\" transform=\"scale(0.015625)\"/>\n","      </defs>\n","      <use xlink:href=\"#DejaVuSans-65\"/>\n","      <use xlink:href=\"#DejaVuSans-70\" x=\"61.523438\"/>\n","      <use xlink:href=\"#DejaVuSans-6f\" x=\"125\"/>\n","      <use xlink:href=\"#DejaVuSans-63\" x=\"186.181641\"/>\n","      <use xlink:href=\"#DejaVuSans-68\" x=\"241.162109\"/>\n","     </g>\n","    </g>\n","   </g>\n","   <g id=\"matplotlib.axis_2\">\n","    <g id=\"ytick_1\">\n","     <g id=\"line2d_11\">\n","      <path d=\"M 30.103125 118.252634 \n","L 225.403125 118.252634 \n","\" clip-path=\"url(#p0c662c7060)\" style=\"fill: none; stroke: #b0b0b0; stroke-width: 0.8; stroke-linecap: square\"/>\n","     </g>\n","     <g id=\"line2d_12\">\n","      <defs>\n","       <path id=\"mcb0c07d6bd\" d=\"M 0 0 \n","L -3.5 0 \n","\" style=\"stroke: #000000; stroke-width: 0.8\"/>\n","      </defs>\n","      <g>\n","       <use xlink:href=\"#mcb0c07d6bd\" x=\"30.103125\" y=\"118.252634\" style=\"stroke: #000000; stroke-width: 0.8\"/>\n","      </g>\n","     </g>\n","     <g id=\"text_7\">\n","      <!-- 0.4 -->\n","      <g transform=\"translate(7.2 122.051853)scale(0.1 -0.1)\">\n","       <defs>\n","        <path id=\"DejaVuSans-2e\" d=\"M 684 794 \n","L 1344 794 \n","L 1344 0 \n","L 684 0 \n","L 684 794 \n","z\n","\" transform=\"scale(0.015625)\"/>\n","       </defs>\n","       <use xlink:href=\"#DejaVuSans-30\"/>\n","       <use xlink:href=\"#DejaVuSans-2e\" x=\"63.623047\"/>\n","       <use xlink:href=\"#DejaVuSans-34\" x=\"95.410156\"/>\n","      </g>\n","     </g>\n","    </g>\n","    <g id=\"ytick_2\">\n","     <g id=\"line2d_13\">\n","      <path d=\"M 30.103125 90.407945 \n","L 225.403125 90.407945 \n","\" clip-path=\"url(#p0c662c7060)\" style=\"fill: none; stroke: #b0b0b0; stroke-width: 0.8; stroke-linecap: square\"/>\n","     </g>\n","     <g id=\"line2d_14\">\n","      <g>\n","       <use xlink:href=\"#mcb0c07d6bd\" x=\"30.103125\" y=\"90.407945\" style=\"stroke: #000000; stroke-width: 0.8\"/>\n","      </g>\n","     </g>\n","     <g id=\"text_8\">\n","      <!-- 0.6 -->\n","      <g transform=\"translate(7.2 94.207164)scale(0.1 -0.1)\">\n","       <use xlink:href=\"#DejaVuSans-30\"/>\n","       <use xlink:href=\"#DejaVuSans-2e\" x=\"63.623047\"/>\n","       <use xlink:href=\"#DejaVuSans-36\" x=\"95.410156\"/>\n","      </g>\n","     </g>\n","    </g>\n","    <g id=\"ytick_3\">\n","     <g id=\"line2d_15\">\n","      <path d=\"M 30.103125 62.563256 \n","L 225.403125 62.563256 \n","\" clip-path=\"url(#p0c662c7060)\" style=\"fill: none; stroke: #b0b0b0; stroke-width: 0.8; stroke-linecap: square\"/>\n","     </g>\n","     <g id=\"line2d_16\">\n","      <g>\n","       <use xlink:href=\"#mcb0c07d6bd\" x=\"30.103125\" y=\"62.563256\" style=\"stroke: #000000; stroke-width: 0.8\"/>\n","      </g>\n","     </g>\n","     <g id=\"text_9\">\n","      <!-- 0.8 -->\n","      <g transform=\"translate(7.2 66.362475)scale(0.1 -0.1)\">\n","       <use xlink:href=\"#DejaVuSans-30\"/>\n","       <use xlink:href=\"#DejaVuSans-2e\" x=\"63.623047\"/>\n","       <use xlink:href=\"#DejaVuSans-38\" x=\"95.410156\"/>\n","      </g>\n","     </g>\n","    </g>\n","    <g id=\"ytick_4\">\n","     <g id=\"line2d_17\">\n","      <path d=\"M 30.103125 34.718567 \n","L 225.403125 34.718567 \n","\" clip-path=\"url(#p0c662c7060)\" style=\"fill: none; stroke: #b0b0b0; stroke-width: 0.8; stroke-linecap: square\"/>\n","     </g>\n","     <g id=\"line2d_18\">\n","      <g>\n","       <use xlink:href=\"#mcb0c07d6bd\" x=\"30.103125\" y=\"34.718567\" style=\"stroke: #000000; stroke-width: 0.8\"/>\n","      </g>\n","     </g>\n","     <g id=\"text_10\">\n","      <!-- 1.0 -->\n","      <g transform=\"translate(7.2 38.517786)scale(0.1 -0.1)\">\n","       <use xlink:href=\"#DejaVuSans-31\"/>\n","       <use xlink:href=\"#DejaVuSans-2e\" x=\"63.623047\"/>\n","       <use xlink:href=\"#DejaVuSans-30\" x=\"95.410156\"/>\n","      </g>\n","     </g>\n","    </g>\n","   </g>\n","   <g id=\"line2d_19\">\n","    <path d=\"M 12.743125 13.377273 \n","L 17.083125 43.767936 \n","L 21.423125 58.417671 \n","L 25.763125 67.479187 \n","L 30.103125 73.262007 \n","L 34.443125 103.663806 \n","L 38.783125 103.763138 \n","L 43.123125 104.394121 \n","L 47.463125 105.582251 \n","L 51.803125 107.229441 \n","L 56.143125 114.076589 \n","L 60.483125 114.849843 \n","L 64.823125 115.446743 \n","L 69.163125 115.742713 \n","L 73.503125 116.210018 \n","L 77.843125 121.422453 \n","L 82.183125 121.75703 \n","L 86.523125 122.117425 \n","L 90.863125 123.000337 \n","L 95.203125 123.120053 \n","L 99.543125 124.154632 \n","L 103.883125 125.988146 \n","L 108.223125 126.436681 \n","L 112.563125 126.391391 \n","L 116.903125 126.592782 \n","L 121.243125 127.180799 \n","L 125.583125 128.857135 \n","L 129.923125 128.954269 \n","L 134.263125 128.972023 \n","L 138.603125 128.942028 \n","L 142.943125 131.348719 \n","L 147.283125 131.034622 \n","L 151.623125 131.254269 \n","L 155.963125 131.468163 \n","L 160.303125 131.690941 \n","L 164.643125 133.073222 \n","L 168.983125 132.942483 \n","L 173.323125 133.310318 \n","L 177.663125 133.263644 \n","L 182.003125 133.419965 \n","L 186.343125 134.529867 \n","L 190.683125 134.642065 \n","L 195.023125 135.228973 \n","L 199.363125 135.361684 \n","L 203.703125 135.169029 \n","L 208.043125 136.922727 \n","L 212.383125 136.094996 \n","L 216.723125 135.716853 \n","L 221.063125 135.993264 \n","L 225.403125 136.605667 \n","\" clip-path=\"url(#p0c662c7060)\" style=\"fill: none; stroke: #1f77b4; stroke-width: 1.5; stroke-linecap: square\"/>\n","   </g>\n","   <g id=\"line2d_20\">\n","    <path d=\"M 12.743125 90.861532 \n","L 17.083125 81.014528 \n","L 21.423125 76.108383 \n","L 25.763125 72.943688 \n","L 30.103125 71.060527 \n","L 34.443125 60.394833 \n","L 38.783125 60.336977 \n","L 43.123125 60.016843 \n","L 47.463125 59.561714 \n","L 51.803125 58.924884 \n","L 56.143125 56.541657 \n","L 60.483125 56.350734 \n","L 64.823125 56.221523 \n","L 69.163125 56.035422 \n","L 73.503125 55.836444 \n","L 77.843125 53.961302 \n","L 82.183125 53.776165 \n","L 86.523125 53.695167 \n","L 90.863125 53.30175 \n","L 95.203125 53.300256 \n","L 99.543125 53.105041 \n","L 103.883125 52.532272 \n","L 108.223125 52.325921 \n","L 112.563125 52.344242 \n","L 116.903125 52.242158 \n","L 121.243125 51.392519 \n","L 125.583125 51.097456 \n","L 129.923125 51.126384 \n","L 134.263125 51.239202 \n","L 138.603125 51.311682 \n","L 142.943125 50.675111 \n","L 147.283125 50.530472 \n","L 151.623125 50.401261 \n","L 155.963125 50.333763 \n","L 160.303125 50.223418 \n","L 164.643125 49.587428 \n","L 168.983125 49.830421 \n","L 173.323125 49.652997 \n","L 177.663125 49.630819 \n","L 182.003125 49.63868 \n","L 186.343125 49.170868 \n","L 190.683125 49.437003 \n","L 195.023125 49.186296 \n","L 199.363125 49.170868 \n","L 203.703125 49.218689 \n","L 208.043125 48.465031 \n","L 212.383125 48.777451 \n","L 216.723125 48.87002 \n","L 221.063125 48.812164 \n","L 225.403125 48.62699 \n","\" clip-path=\"url(#p0c662c7060)\" style=\"fill: none; stroke-dasharray: 5.55,2.4; stroke-dashoffset: 0; stroke: #bf00bf; stroke-width: 1.5\"/>\n","   </g>\n","   <g id=\"line2d_21\">\n","    <path d=\"M 30.103125 73.436607 \n","L 51.803125 58.665 \n","L 73.503125 63.621354 \n","L 95.203125 85.381979 \n","L 116.903125 64.108637 \n","L 138.603125 55.685618 \n","L 160.303125 58.567543 \n","L 182.003125 51.522837 \n","L 203.703125 69.051069 \n","L 225.403125 58.316941 \n","\" clip-path=\"url(#p0c662c7060)\" style=\"fill: none; stroke-dasharray: 9.6,2.4,1.5,2.4; stroke-dashoffset: 0; stroke: #008000; stroke-width: 1.5\"/>\n","   </g>\n","   <g id=\"patch_3\">\n","    <path d=\"M 30.103125 143.1 \n","L 30.103125 7.2 \n","\" style=\"fill: none; stroke: #000000; stroke-width: 0.8; stroke-linejoin: miter; stroke-linecap: square\"/>\n","   </g>\n","   <g id=\"patch_4\">\n","    <path d=\"M 225.403125 143.1 \n","L 225.403125 7.2 \n","\" style=\"fill: none; stroke: #000000; stroke-width: 0.8; stroke-linejoin: miter; stroke-linecap: square\"/>\n","   </g>\n","   <g id=\"patch_5\">\n","    <path d=\"M 30.103125 143.1 \n","L 225.403125 143.1 \n","\" style=\"fill: none; stroke: #000000; stroke-width: 0.8; stroke-linejoin: miter; stroke-linecap: square\"/>\n","   </g>\n","   <g id=\"patch_6\">\n","    <path d=\"M 30.103125 7.2 \n","L 225.403125 7.2 \n","\" style=\"fill: none; stroke: #000000; stroke-width: 0.8; stroke-linejoin: miter; stroke-linecap: square\"/>\n","   </g>\n","   <g id=\"legend_1\">\n","    <g id=\"patch_7\">\n","     <path d=\"M 140.634375 98.667187 \n","L 218.403125 98.667187 \n","Q 220.403125 98.667187 220.403125 96.667187 \n","L 220.403125 53.632812 \n","Q 220.403125 51.632812 218.403125 51.632812 \n","L 140.634375 51.632812 \n","Q 138.634375 51.632812 138.634375 53.632812 \n","L 138.634375 96.667187 \n","Q 138.634375 98.667187 140.634375 98.667187 \n","z\n","\" style=\"fill: #ffffff; opacity: 0.8; stroke: #cccccc; stroke-linejoin: miter\"/>\n","    </g>\n","    <g id=\"line2d_22\">\n","     <path d=\"M 142.634375 59.73125 \n","L 152.634375 59.73125 \n","L 162.634375 59.73125 \n","\" style=\"fill: none; stroke: #1f77b4; stroke-width: 1.5; stroke-linecap: square\"/>\n","    </g>\n","    <g id=\"text_11\">\n","     <!-- train loss -->\n","     <g transform=\"translate(170.634375 63.23125)scale(0.1 -0.1)\">\n","      <defs>\n","       <path id=\"DejaVuSans-74\" d=\"M 1172 4494 \n","L 1172 3500 \n","L 2356 3500 \n","L 2356 3053 \n","L 1172 3053 \n","L 1172 1153 \n","Q 1172 725 1289 603 \n","Q 1406 481 1766 481 \n","L 2356 481 \n","L 2356 0 \n","L 1766 0 \n","Q 1100 0 847 248 \n","Q 594 497 594 1153 \n","L 594 3053 \n","L 172 3053 \n","L 172 3500 \n","L 594 3500 \n","L 594 4494 \n","L 1172 4494 \n","z\n","\" transform=\"scale(0.015625)\"/>\n","       <path id=\"DejaVuSans-72\" d=\"M 2631 2963 \n","Q 2534 3019 2420 3045 \n","Q 2306 3072 2169 3072 \n","Q 1681 3072 1420 2755 \n","Q 1159 2438 1159 1844 \n","L 1159 0 \n","L 581 0 \n","L 581 3500 \n","L 1159 3500 \n","L 1159 2956 \n","Q 1341 3275 1631 3429 \n","Q 1922 3584 2338 3584 \n","Q 2397 3584 2469 3576 \n","Q 2541 3569 2628 3553 \n","L 2631 2963 \n","z\n","\" transform=\"scale(0.015625)\"/>\n","       <path id=\"DejaVuSans-61\" d=\"M 2194 1759 \n","Q 1497 1759 1228 1600 \n","Q 959 1441 959 1056 \n","Q 959 750 1161 570 \n","Q 1363 391 1709 391 \n","Q 2188 391 2477 730 \n","Q 2766 1069 2766 1631 \n","L 2766 1759 \n","L 2194 1759 \n","z\n","M 3341 1997 \n","L 3341 0 \n","L 2766 0 \n","L 2766 531 \n","Q 2569 213 2275 61 \n","Q 1981 -91 1556 -91 \n","Q 1019 -91 701 211 \n","Q 384 513 384 1019 \n","Q 384 1609 779 1909 \n","Q 1175 2209 1959 2209 \n","L 2766 2209 \n","L 2766 2266 \n","Q 2766 2663 2505 2880 \n","Q 2244 3097 1772 3097 \n","Q 1472 3097 1187 3025 \n","Q 903 2953 641 2809 \n","L 641 3341 \n","Q 956 3463 1253 3523 \n","Q 1550 3584 1831 3584 \n","Q 2591 3584 2966 3190 \n","Q 3341 2797 3341 1997 \n","z\n","\" transform=\"scale(0.015625)\"/>\n","       <path id=\"DejaVuSans-69\" d=\"M 603 3500 \n","L 1178 3500 \n","L 1178 0 \n","L 603 0 \n","L 603 3500 \n","z\n","M 603 4863 \n","L 1178 4863 \n","L 1178 4134 \n","L 603 4134 \n","L 603 4863 \n","z\n","\" transform=\"scale(0.015625)\"/>\n","       <path id=\"DejaVuSans-6e\" d=\"M 3513 2113 \n","L 3513 0 \n","L 2938 0 \n","L 2938 2094 \n","Q 2938 2591 2744 2837 \n","Q 2550 3084 2163 3084 \n","Q 1697 3084 1428 2787 \n","Q 1159 2491 1159 1978 \n","L 1159 0 \n","L 581 0 \n","L 581 3500 \n","L 1159 3500 \n","L 1159 2956 \n","Q 1366 3272 1645 3428 \n","Q 1925 3584 2291 3584 \n","Q 2894 3584 3203 3211 \n","Q 3513 2838 3513 2113 \n","z\n","\" transform=\"scale(0.015625)\"/>\n","       <path id=\"DejaVuSans-20\" transform=\"scale(0.015625)\"/>\n","       <path id=\"DejaVuSans-6c\" d=\"M 603 4863 \n","L 1178 4863 \n","L 1178 0 \n","L 603 0 \n","L 603 4863 \n","z\n","\" transform=\"scale(0.015625)\"/>\n","       <path id=\"DejaVuSans-73\" d=\"M 2834 3397 \n","L 2834 2853 \n","Q 2591 2978 2328 3040 \n","Q 2066 3103 1784 3103 \n","Q 1356 3103 1142 2972 \n","Q 928 2841 928 2578 \n","Q 928 2378 1081 2264 \n","Q 1234 2150 1697 2047 \n","L 1894 2003 \n","Q 2506 1872 2764 1633 \n","Q 3022 1394 3022 966 \n","Q 3022 478 2636 193 \n","Q 2250 -91 1575 -91 \n","Q 1294 -91 989 -36 \n","Q 684 19 347 128 \n","L 347 722 \n","Q 666 556 975 473 \n","Q 1284 391 1588 391 \n","Q 1994 391 2212 530 \n","Q 2431 669 2431 922 \n","Q 2431 1156 2273 1281 \n","Q 2116 1406 1581 1522 \n","L 1381 1569 \n","Q 847 1681 609 1914 \n","Q 372 2147 372 2553 \n","Q 372 3047 722 3315 \n","Q 1072 3584 1716 3584 \n","Q 2034 3584 2315 3537 \n","Q 2597 3491 2834 3397 \n","z\n","\" transform=\"scale(0.015625)\"/>\n","      </defs>\n","      <use xlink:href=\"#DejaVuSans-74\"/>\n","      <use xlink:href=\"#DejaVuSans-72\" x=\"39.208984\"/>\n","      <use xlink:href=\"#DejaVuSans-61\" x=\"80.322266\"/>\n","      <use xlink:href=\"#DejaVuSans-69\" x=\"141.601562\"/>\n","      <use xlink:href=\"#DejaVuSans-6e\" x=\"169.384766\"/>\n","      <use xlink:href=\"#DejaVuSans-20\" x=\"232.763672\"/>\n","      <use xlink:href=\"#DejaVuSans-6c\" x=\"264.550781\"/>\n","      <use xlink:href=\"#DejaVuSans-6f\" x=\"292.333984\"/>\n","      <use xlink:href=\"#DejaVuSans-73\" x=\"353.515625\"/>\n","      <use xlink:href=\"#DejaVuSans-73\" x=\"405.615234\"/>\n","     </g>\n","    </g>\n","    <g id=\"line2d_23\">\n","     <path d=\"M 142.634375 74.409375 \n","L 152.634375 74.409375 \n","L 162.634375 74.409375 \n","\" style=\"fill: none; stroke-dasharray: 5.55,2.4; stroke-dashoffset: 0; stroke: #bf00bf; stroke-width: 1.5\"/>\n","    </g>\n","    <g id=\"text_12\">\n","     <!-- train acc -->\n","     <g transform=\"translate(170.634375 77.909375)scale(0.1 -0.1)\">\n","      <use xlink:href=\"#DejaVuSans-74\"/>\n","      <use xlink:href=\"#DejaVuSans-72\" x=\"39.208984\"/>\n","      <use xlink:href=\"#DejaVuSans-61\" x=\"80.322266\"/>\n","      <use xlink:href=\"#DejaVuSans-69\" x=\"141.601562\"/>\n","      <use xlink:href=\"#DejaVuSans-6e\" x=\"169.384766\"/>\n","      <use xlink:href=\"#DejaVuSans-20\" x=\"232.763672\"/>\n","      <use xlink:href=\"#DejaVuSans-61\" x=\"264.550781\"/>\n","      <use xlink:href=\"#DejaVuSans-63\" x=\"325.830078\"/>\n","      <use xlink:href=\"#DejaVuSans-63\" x=\"380.810547\"/>\n","     </g>\n","    </g>\n","    <g id=\"line2d_24\">\n","     <path d=\"M 142.634375 89.0875 \n","L 152.634375 89.0875 \n","L 162.634375 89.0875 \n","\" style=\"fill: none; stroke-dasharray: 9.6,2.4,1.5,2.4; stroke-dashoffset: 0; stroke: #008000; stroke-width: 1.5\"/>\n","    </g>\n","    <g id=\"text_13\">\n","     <!-- test acc -->\n","     <g transform=\"translate(170.634375 92.5875)scale(0.1 -0.1)\">\n","      <use xlink:href=\"#DejaVuSans-74\"/>\n","      <use xlink:href=\"#DejaVuSans-65\" x=\"39.208984\"/>\n","      <use xlink:href=\"#DejaVuSans-73\" x=\"100.732422\"/>\n","      <use xlink:href=\"#DejaVuSans-74\" x=\"152.832031\"/>\n","      <use xlink:href=\"#DejaVuSans-20\" x=\"192.041016\"/>\n","      <use xlink:href=\"#DejaVuSans-61\" x=\"223.828125\"/>\n","      <use xlink:href=\"#DejaVuSans-63\" x=\"285.107422\"/>\n","      <use xlink:href=\"#DejaVuSans-63\" x=\"340.087891\"/>\n","     </g>\n","    </g>\n","   </g>\n","  </g>\n"," </g>\n"," <defs>\n","  <clipPath id=\"p0c662c7060\">\n","   <rect x=\"30.103125\" y=\"7.2\" width=\"195.3\" height=\"135.9\"/>\n","  </clipPath>\n"," </defs>\n","</svg>\n"],"text/plain":["<Figure size 252x180 with 1 Axes>"]},"metadata":{"needs_background":"light"},"output_type":"display_data"}],"source":["lr, num_epochs, batch_size = 1.0, 10, 256\n","train_iter, test_iter = d2l.load_data_fashion_mnist(batch_size)\n","d2l.train_ch6(net, train_iter, test_iter, num_epochs, lr, d2l.try_gpu())"],"id":"a19ac3fc"},{"cell_type":"markdown","metadata":{"origin_pos":15,"id":"a175154a"},"source":["让我们来看看从第一个批量规范化层中学到的[**拉伸参数`gamma`和偏移参数`beta`**]。\n"],"id":"a175154a"},{"cell_type":"code","execution_count":null,"metadata":{"execution":{"iopub.execute_input":"2022-07-31T02:30:38.984449Z","iopub.status.busy":"2022-07-31T02:30:38.984236Z","iopub.status.idle":"2022-07-31T02:30:38.995863Z","shell.execute_reply":"2022-07-31T02:30:38.995216Z"},"origin_pos":17,"tab":["pytorch"],"id":"a348127f","outputId":"e8202795-14ff-4b10-aaa4-8875d7a90f51"},"outputs":[{"data":{"text/plain":["(tensor([0.3362, 4.0349, 0.4496, 3.7056, 3.7774, 2.6762], device='cuda:0',\n","        grad_fn=<ReshapeAliasBackward0>),\n"," tensor([-0.5739,  4.1376,  0.5126,  0.3060, -2.5187,  0.3683], device='cuda:0',\n","        grad_fn=<ReshapeAliasBackward0>))"]},"execution_count":5,"metadata":{},"output_type":"execute_result"}],"source":["net[1].gamma.reshape((-1,)), net[1].beta.reshape((-1,))"],"id":"a348127f"},{"cell_type":"markdown","metadata":{"origin_pos":19,"id":"5aaba5ec"},"source":["## [**简明实现**]\n","\n","除了使用我们刚刚定义的`BatchNorm`，我们也可以直接使用深度学习框架中定义的`BatchNorm`。\n","该代码看起来几乎与我们上面的代码相同。\n"],"id":"5aaba5ec"},{"cell_type":"code","execution_count":null,"metadata":{"execution":{"iopub.execute_input":"2022-07-31T02:30:38.999650Z","iopub.status.busy":"2022-07-31T02:30:38.999443Z","iopub.status.idle":"2022-07-31T02:30:39.007161Z","shell.execute_reply":"2022-07-31T02:30:39.006497Z"},"origin_pos":21,"tab":["pytorch"],"id":"e0b4358a"},"outputs":[],"source":["net = nn.Sequential(\n","    nn.Conv2d(1, 6, kernel_size=5), nn.BatchNorm2d(6), nn.Sigmoid(),\n","    nn.AvgPool2d(kernel_size=2, stride=2),\n","    nn.Conv2d(6, 16, kernel_size=5), nn.BatchNorm2d(16), nn.Sigmoid(),\n","    nn.AvgPool2d(kernel_size=2, stride=2), nn.Flatten(),\n","    nn.Linear(256, 120), nn.BatchNorm1d(120), nn.Sigmoid(),\n","    nn.Linear(120, 84), nn.BatchNorm1d(84), nn.Sigmoid(),\n","    nn.Linear(84, 10))"],"id":"e0b4358a"},{"cell_type":"markdown","metadata":{"origin_pos":23,"id":"f67a650e"},"source":["下面，我们[**使用相同超参数来训练模型**]。\n","请注意，通常高级API变体运行速度快得多，因为它的代码已编译为C++或CUDA，而我们的自定义代码由Python实现。\n"],"id":"f67a650e"},{"cell_type":"code","execution_count":null,"metadata":{"execution":{"iopub.execute_input":"2022-07-31T02:30:39.010920Z","iopub.status.busy":"2022-07-31T02:30:39.010458Z","iopub.status.idle":"2022-07-31T02:31:21.257015Z","shell.execute_reply":"2022-07-31T02:31:21.256292Z"},"origin_pos":24,"tab":["pytorch"],"id":"1fa34720","outputId":"583fb255-2f0e-4612-e659-992fab5f686a"},"outputs":[{"name":"stdout","output_type":"stream","text":["loss 0.269, train acc 0.901, test acc 0.853\n","64557.2 examples/sec on cuda:0\n"]},{"data":{"image/svg+xml":["<?xml version=\"1.0\" encoding=\"utf-8\" standalone=\"no\"?>\n","<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\n","  \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\n","<svg xmlns:xlink=\"http://www.w3.org/1999/xlink\" width=\"238.965625pt\" height=\"181.526186pt\" viewBox=\"0 0 238.965625 181.526186\" xmlns=\"http://www.w3.org/2000/svg\" version=\"1.1\">\n"," <metadata>\n","  <rdf:RDF xmlns:dc=\"http://purl.org/dc/elements/1.1/\" xmlns:cc=\"http://creativecommons.org/ns#\" xmlns:rdf=\"http://www.w3.org/1999/02/22-rdf-syntax-ns#\">\n","   <cc:Work>\n","    <dc:type rdf:resource=\"http://purl.org/dc/dcmitype/StillImage\"/>\n","    <dc:date>2022-07-31T02:31:21.211085</dc:date>\n","    <dc:format>image/svg+xml</dc:format>\n","    <dc:creator>\n","     <cc:Agent>\n","      <dc:title>Matplotlib v3.5.1, https://matplotlib.org/</dc:title>\n","     </cc:Agent>\n","    </dc:creator>\n","   </cc:Work>\n","  </rdf:RDF>\n"," </metadata>\n"," <defs>\n","  <style type=\"text/css\">*{stroke-linejoin: round; stroke-linecap: butt}</style>\n"," </defs>\n"," <g id=\"figure_1\">\n","  <g id=\"patch_1\">\n","   <path d=\"M 0 181.526186 \n","L 238.965625 181.526186 \n","L 238.965625 0 \n","L 0 0 \n","L 0 181.526186 \n","z\n","\" style=\"fill: none\"/>\n","  </g>\n","  <g id=\"axes_1\">\n","   <g id=\"patch_2\">\n","    <path d=\"M 30.103125 143.969936 \n","L 225.403125 143.969936 \n","L 225.403125 8.069936 \n","L 30.103125 8.069936 \n","z\n","\" style=\"fill: #ffffff\"/>\n","   </g>\n","   <g id=\"matplotlib.axis_1\">\n","    <g id=\"xtick_1\">\n","     <g id=\"line2d_1\">\n","      <path d=\"M 51.803125 143.969936 \n","L 51.803125 8.069936 \n","\" clip-path=\"url(#p0bf9a0d35c)\" style=\"fill: none; stroke: #b0b0b0; stroke-width: 0.8; stroke-linecap: square\"/>\n","     </g>\n","     <g id=\"line2d_2\">\n","      <defs>\n","       <path id=\"m41472d80b0\" d=\"M 0 0 \n","L 0 3.5 \n","\" style=\"stroke: #000000; stroke-width: 0.8\"/>\n","      </defs>\n","      <g>\n","       <use xlink:href=\"#m41472d80b0\" x=\"51.803125\" y=\"143.969936\" style=\"stroke: #000000; stroke-width: 0.8\"/>\n","      </g>\n","     </g>\n","     <g id=\"text_1\">\n","      <!-- 2 -->\n","      <g transform=\"translate(48.621875 158.568374)scale(0.1 -0.1)\">\n","       <defs>\n","        <path id=\"DejaVuSans-32\" d=\"M 1228 531 \n","L 3431 531 \n","L 3431 0 \n","L 469 0 \n","L 469 531 \n","Q 828 903 1448 1529 \n","Q 2069 2156 2228 2338 \n","Q 2531 2678 2651 2914 \n","Q 2772 3150 2772 3378 \n","Q 2772 3750 2511 3984 \n","Q 2250 4219 1831 4219 \n","Q 1534 4219 1204 4116 \n","Q 875 4013 500 3803 \n","L 500 4441 \n","Q 881 4594 1212 4672 \n","Q 1544 4750 1819 4750 \n","Q 2544 4750 2975 4387 \n","Q 3406 4025 3406 3419 \n","Q 3406 3131 3298 2873 \n","Q 3191 2616 2906 2266 \n","Q 2828 2175 2409 1742 \n","Q 1991 1309 1228 531 \n","z\n","\" transform=\"scale(0.015625)\"/>\n","       </defs>\n","       <use xlink:href=\"#DejaVuSans-32\"/>\n","      </g>\n","     </g>\n","    </g>\n","    <g id=\"xtick_2\">\n","     <g id=\"line2d_3\">\n","      <path d=\"M 95.203125 143.969936 \n","L 95.203125 8.069936 \n","\" clip-path=\"url(#p0bf9a0d35c)\" style=\"fill: none; stroke: #b0b0b0; stroke-width: 0.8; stroke-linecap: square\"/>\n","     </g>\n","     <g id=\"line2d_4\">\n","      <g>\n","       <use xlink:href=\"#m41472d80b0\" x=\"95.203125\" y=\"143.969936\" style=\"stroke: #000000; stroke-width: 0.8\"/>\n","      </g>\n","     </g>\n","     <g id=\"text_2\">\n","      <!-- 4 -->\n","      <g transform=\"translate(92.021875 158.568374)scale(0.1 -0.1)\">\n","       <defs>\n","        <path id=\"DejaVuSans-34\" d=\"M 2419 4116 \n","L 825 1625 \n","L 2419 1625 \n","L 2419 4116 \n","z\n","M 2253 4666 \n","L 3047 4666 \n","L 3047 1625 \n","L 3713 1625 \n","L 3713 1100 \n","L 3047 1100 \n","L 3047 0 \n","L 2419 0 \n","L 2419 1100 \n","L 313 1100 \n","L 313 1709 \n","L 2253 4666 \n","z\n","\" transform=\"scale(0.015625)\"/>\n","       </defs>\n","       <use xlink:href=\"#DejaVuSans-34\"/>\n","      </g>\n","     </g>\n","    </g>\n","    <g id=\"xtick_3\">\n","     <g id=\"line2d_5\">\n","      <path d=\"M 138.603125 143.969936 \n","L 138.603125 8.069936 \n","\" clip-path=\"url(#p0bf9a0d35c)\" style=\"fill: none; stroke: #b0b0b0; stroke-width: 0.8; stroke-linecap: square\"/>\n","     </g>\n","     <g id=\"line2d_6\">\n","      <g>\n","       <use xlink:href=\"#m41472d80b0\" x=\"138.603125\" y=\"143.969936\" style=\"stroke: #000000; stroke-width: 0.8\"/>\n","      </g>\n","     </g>\n","     <g id=\"text_3\">\n","      <!-- 6 -->\n","      <g transform=\"translate(135.421875 158.568374)scale(0.1 -0.1)\">\n","       <defs>\n","        <path id=\"DejaVuSans-36\" d=\"M 2113 2584 \n","Q 1688 2584 1439 2293 \n","Q 1191 2003 1191 1497 \n","Q 1191 994 1439 701 \n","Q 1688 409 2113 409 \n","Q 2538 409 2786 701 \n","Q 3034 994 3034 1497 \n","Q 3034 2003 2786 2293 \n","Q 2538 2584 2113 2584 \n","z\n","M 3366 4563 \n","L 3366 3988 \n","Q 3128 4100 2886 4159 \n","Q 2644 4219 2406 4219 \n","Q 1781 4219 1451 3797 \n","Q 1122 3375 1075 2522 \n","Q 1259 2794 1537 2939 \n","Q 1816 3084 2150 3084 \n","Q 2853 3084 3261 2657 \n","Q 3669 2231 3669 1497 \n","Q 3669 778 3244 343 \n","Q 2819 -91 2113 -91 \n","Q 1303 -91 875 529 \n","Q 447 1150 447 2328 \n","Q 447 3434 972 4092 \n","Q 1497 4750 2381 4750 \n","Q 2619 4750 2861 4703 \n","Q 3103 4656 3366 4563 \n","z\n","\" transform=\"scale(0.015625)\"/>\n","       </defs>\n","       <use xlink:href=\"#DejaVuSans-36\"/>\n","      </g>\n","     </g>\n","    </g>\n","    <g id=\"xtick_4\">\n","     <g id=\"line2d_7\">\n","      <path d=\"M 182.003125 143.969936 \n","L 182.003125 8.069936 \n","\" clip-path=\"url(#p0bf9a0d35c)\" style=\"fill: none; stroke: #b0b0b0; stroke-width: 0.8; stroke-linecap: square\"/>\n","     </g>\n","     <g id=\"line2d_8\">\n","      <g>\n","       <use xlink:href=\"#m41472d80b0\" x=\"182.003125\" y=\"143.969936\" style=\"stroke: #000000; stroke-width: 0.8\"/>\n","      </g>\n","     </g>\n","     <g id=\"text_4\">\n","      <!-- 8 -->\n","      <g transform=\"translate(178.821875 158.568374)scale(0.1 -0.1)\">\n","       <defs>\n","        <path id=\"DejaVuSans-38\" d=\"M 2034 2216 \n","Q 1584 2216 1326 1975 \n","Q 1069 1734 1069 1313 \n","Q 1069 891 1326 650 \n","Q 1584 409 2034 409 \n","Q 2484 409 2743 651 \n","Q 3003 894 3003 1313 \n","Q 3003 1734 2745 1975 \n","Q 2488 2216 2034 2216 \n","z\n","M 1403 2484 \n","Q 997 2584 770 2862 \n","Q 544 3141 544 3541 \n","Q 544 4100 942 4425 \n","Q 1341 4750 2034 4750 \n","Q 2731 4750 3128 4425 \n","Q 3525 4100 3525 3541 \n","Q 3525 3141 3298 2862 \n","Q 3072 2584 2669 2484 \n","Q 3125 2378 3379 2068 \n","Q 3634 1759 3634 1313 \n","Q 3634 634 3220 271 \n","Q 2806 -91 2034 -91 \n","Q 1263 -91 848 271 \n","Q 434 634 434 1313 \n","Q 434 1759 690 2068 \n","Q 947 2378 1403 2484 \n","z\n","M 1172 3481 \n","Q 1172 3119 1398 2916 \n","Q 1625 2713 2034 2713 \n","Q 2441 2713 2670 2916 \n","Q 2900 3119 2900 3481 \n","Q 2900 3844 2670 4047 \n","Q 2441 4250 2034 4250 \n","Q 1625 4250 1398 4047 \n","Q 1172 3844 1172 3481 \n","z\n","\" transform=\"scale(0.015625)\"/>\n","       </defs>\n","       <use xlink:href=\"#DejaVuSans-38\"/>\n","      </g>\n","     </g>\n","    </g>\n","    <g id=\"xtick_5\">\n","     <g id=\"line2d_9\">\n","      <path d=\"M 225.403125 143.969936 \n","L 225.403125 8.069936 \n","\" clip-path=\"url(#p0bf9a0d35c)\" style=\"fill: none; stroke: #b0b0b0; stroke-width: 0.8; stroke-linecap: square\"/>\n","     </g>\n","     <g id=\"line2d_10\">\n","      <g>\n","       <use xlink:href=\"#m41472d80b0\" x=\"225.403125\" y=\"143.969936\" style=\"stroke: #000000; stroke-width: 0.8\"/>\n","      </g>\n","     </g>\n","     <g id=\"text_5\">\n","      <!-- 10 -->\n","      <g transform=\"translate(219.040625 158.568374)scale(0.1 -0.1)\">\n","       <defs>\n","        <path id=\"DejaVuSans-31\" d=\"M 794 531 \n","L 1825 531 \n","L 1825 4091 \n","L 703 3866 \n","L 703 4441 \n","L 1819 4666 \n","L 2450 4666 \n","L 2450 531 \n","L 3481 531 \n","L 3481 0 \n","L 794 0 \n","L 794 531 \n","z\n","\" transform=\"scale(0.015625)\"/>\n","        <path id=\"DejaVuSans-30\" d=\"M 2034 4250 \n","Q 1547 4250 1301 3770 \n","Q 1056 3291 1056 2328 \n","Q 1056 1369 1301 889 \n","Q 1547 409 2034 409 \n","Q 2525 409 2770 889 \n","Q 3016 1369 3016 2328 \n","Q 3016 3291 2770 3770 \n","Q 2525 4250 2034 4250 \n","z\n","M 2034 4750 \n","Q 2819 4750 3233 4129 \n","Q 3647 3509 3647 2328 \n","Q 3647 1150 3233 529 \n","Q 2819 -91 2034 -91 \n","Q 1250 -91 836 529 \n","Q 422 1150 422 2328 \n","Q 422 3509 836 4129 \n","Q 1250 4750 2034 4750 \n","z\n","\" transform=\"scale(0.015625)\"/>\n","       </defs>\n","       <use xlink:href=\"#DejaVuSans-31\"/>\n","       <use xlink:href=\"#DejaVuSans-30\" x=\"63.623047\"/>\n","      </g>\n","     </g>\n","    </g>\n","    <g id=\"text_6\">\n","     <!-- epoch -->\n","     <g transform=\"translate(112.525 172.246499)scale(0.1 -0.1)\">\n","      <defs>\n","       <path id=\"DejaVuSans-65\" d=\"M 3597 1894 \n","L 3597 1613 \n","L 953 1613 \n","Q 991 1019 1311 708 \n","Q 1631 397 2203 397 \n","Q 2534 397 2845 478 \n","Q 3156 559 3463 722 \n","L 3463 178 \n","Q 3153 47 2828 -22 \n","Q 2503 -91 2169 -91 \n","Q 1331 -91 842 396 \n","Q 353 884 353 1716 \n","Q 353 2575 817 3079 \n","Q 1281 3584 2069 3584 \n","Q 2775 3584 3186 3129 \n","Q 3597 2675 3597 1894 \n","z\n","M 3022 2063 \n","Q 3016 2534 2758 2815 \n","Q 2500 3097 2075 3097 \n","Q 1594 3097 1305 2825 \n","Q 1016 2553 972 2059 \n","L 3022 2063 \n","z\n","\" transform=\"scale(0.015625)\"/>\n","       <path id=\"DejaVuSans-70\" d=\"M 1159 525 \n","L 1159 -1331 \n","L 581 -1331 \n","L 581 3500 \n","L 1159 3500 \n","L 1159 2969 \n","Q 1341 3281 1617 3432 \n","Q 1894 3584 2278 3584 \n","Q 2916 3584 3314 3078 \n","Q 3713 2572 3713 1747 \n","Q 3713 922 3314 415 \n","Q 2916 -91 2278 -91 \n","Q 1894 -91 1617 61 \n","Q 1341 213 1159 525 \n","z\n","M 3116 1747 \n","Q 3116 2381 2855 2742 \n","Q 2594 3103 2138 3103 \n","Q 1681 3103 1420 2742 \n","Q 1159 2381 1159 1747 \n","Q 1159 1113 1420 752 \n","Q 1681 391 2138 391 \n","Q 2594 391 2855 752 \n","Q 3116 1113 3116 1747 \n","z\n","\" transform=\"scale(0.015625)\"/>\n","       <path id=\"DejaVuSans-6f\" d=\"M 1959 3097 \n","Q 1497 3097 1228 2736 \n","Q 959 2375 959 1747 \n","Q 959 1119 1226 758 \n","Q 1494 397 1959 397 \n","Q 2419 397 2687 759 \n","Q 2956 1122 2956 1747 \n","Q 2956 2369 2687 2733 \n","Q 2419 3097 1959 3097 \n","z\n","M 1959 3584 \n","Q 2709 3584 3137 3096 \n","Q 3566 2609 3566 1747 \n","Q 3566 888 3137 398 \n","Q 2709 -91 1959 -91 \n","Q 1206 -91 779 398 \n","Q 353 888 353 1747 \n","Q 353 2609 779 3096 \n","Q 1206 3584 1959 3584 \n","z\n","\" transform=\"scale(0.015625)\"/>\n","       <path id=\"DejaVuSans-63\" d=\"M 3122 3366 \n","L 3122 2828 \n","Q 2878 2963 2633 3030 \n","Q 2388 3097 2138 3097 \n","Q 1578 3097 1268 2742 \n","Q 959 2388 959 1747 \n","Q 959 1106 1268 751 \n","Q 1578 397 2138 397 \n","Q 2388 397 2633 464 \n","Q 2878 531 3122 666 \n","L 3122 134 \n","Q 2881 22 2623 -34 \n","Q 2366 -91 2075 -91 \n","Q 1284 -91 818 406 \n","Q 353 903 353 1747 \n","Q 353 2603 823 3093 \n","Q 1294 3584 2113 3584 \n","Q 2378 3584 2631 3529 \n","Q 2884 3475 3122 3366 \n","z\n","\" transform=\"scale(0.015625)\"/>\n","       <path id=\"DejaVuSans-68\" d=\"M 3513 2113 \n","L 3513 0 \n","L 2938 0 \n","L 2938 2094 \n","Q 2938 2591 2744 2837 \n","Q 2550 3084 2163 3084 \n","Q 1697 3084 1428 2787 \n","Q 1159 2491 1159 1978 \n","L 1159 0 \n","L 581 0 \n","L 581 4863 \n","L 1159 4863 \n","L 1159 2956 \n","Q 1366 3272 1645 3428 \n","Q 1925 3584 2291 3584 \n","Q 2894 3584 3203 3211 \n","Q 3513 2838 3513 2113 \n","z\n","\" transform=\"scale(0.015625)\"/>\n","      </defs>\n","      <use xlink:href=\"#DejaVuSans-65\"/>\n","      <use xlink:href=\"#DejaVuSans-70\" x=\"61.523438\"/>\n","      <use xlink:href=\"#DejaVuSans-6f\" x=\"125\"/>\n","      <use xlink:href=\"#DejaVuSans-63\" x=\"186.181641\"/>\n","      <use xlink:href=\"#DejaVuSans-68\" x=\"241.162109\"/>\n","     </g>\n","    </g>\n","   </g>\n","   <g id=\"matplotlib.axis_2\">\n","    <g id=\"ytick_1\">\n","     <g id=\"line2d_11\">\n","      <path d=\"M 30.103125 119.297916 \n","L 225.403125 119.297916 \n","\" clip-path=\"url(#p0bf9a0d35c)\" style=\"fill: none; stroke: #b0b0b0; stroke-width: 0.8; stroke-linecap: square\"/>\n","     </g>\n","     <g id=\"line2d_12\">\n","      <defs>\n","       <path id=\"m6ce9d77b77\" d=\"M 0 0 \n","L -3.5 0 \n","\" style=\"stroke: #000000; stroke-width: 0.8\"/>\n","      </defs>\n","      <g>\n","       <use xlink:href=\"#m6ce9d77b77\" x=\"30.103125\" y=\"119.297916\" style=\"stroke: #000000; stroke-width: 0.8\"/>\n","      </g>\n","     </g>\n","     <g id=\"text_7\">\n","      <!-- 0.4 -->\n","      <g transform=\"translate(7.2 123.097135)scale(0.1 -0.1)\">\n","       <defs>\n","        <path id=\"DejaVuSans-2e\" d=\"M 684 794 \n","L 1344 794 \n","L 1344 0 \n","L 684 0 \n","L 684 794 \n","z\n","\" transform=\"scale(0.015625)\"/>\n","       </defs>\n","       <use xlink:href=\"#DejaVuSans-30\"/>\n","       <use xlink:href=\"#DejaVuSans-2e\" x=\"63.623047\"/>\n","       <use xlink:href=\"#DejaVuSans-34\" x=\"95.410156\"/>\n","      </g>\n","     </g>\n","    </g>\n","    <g id=\"ytick_2\">\n","     <g id=\"line2d_13\">\n","      <path d=\"M 30.103125 92.223242 \n","L 225.403125 92.223242 \n","\" clip-path=\"url(#p0bf9a0d35c)\" style=\"fill: none; stroke: #b0b0b0; stroke-width: 0.8; stroke-linecap: square\"/>\n","     </g>\n","     <g id=\"line2d_14\">\n","      <g>\n","       <use xlink:href=\"#m6ce9d77b77\" x=\"30.103125\" y=\"92.223242\" style=\"stroke: #000000; stroke-width: 0.8\"/>\n","      </g>\n","     </g>\n","     <g id=\"text_8\">\n","      <!-- 0.6 -->\n","      <g transform=\"translate(7.2 96.02246)scale(0.1 -0.1)\">\n","       <use xlink:href=\"#DejaVuSans-30\"/>\n","       <use xlink:href=\"#DejaVuSans-2e\" x=\"63.623047\"/>\n","       <use xlink:href=\"#DejaVuSans-36\" x=\"95.410156\"/>\n","      </g>\n","     </g>\n","    </g>\n","    <g id=\"ytick_3\">\n","     <g id=\"line2d_15\">\n","      <path d=\"M 30.103125 65.148567 \n","L 225.403125 65.148567 \n","\" clip-path=\"url(#p0bf9a0d35c)\" style=\"fill: none; stroke: #b0b0b0; stroke-width: 0.8; stroke-linecap: square\"/>\n","     </g>\n","     <g id=\"line2d_16\">\n","      <g>\n","       <use xlink:href=\"#m6ce9d77b77\" x=\"30.103125\" y=\"65.148567\" style=\"stroke: #000000; stroke-width: 0.8\"/>\n","      </g>\n","     </g>\n","     <g id=\"text_9\">\n","      <!-- 0.8 -->\n","      <g transform=\"translate(7.2 68.947786)scale(0.1 -0.1)\">\n","       <use xlink:href=\"#DejaVuSans-30\"/>\n","       <use xlink:href=\"#DejaVuSans-2e\" x=\"63.623047\"/>\n","       <use xlink:href=\"#DejaVuSans-38\" x=\"95.410156\"/>\n","      </g>\n","     </g>\n","    </g>\n","    <g id=\"ytick_4\">\n","     <g id=\"line2d_17\">\n","      <path d=\"M 30.103125 38.073893 \n","L 225.403125 38.073893 \n","\" clip-path=\"url(#p0bf9a0d35c)\" style=\"fill: none; stroke: #b0b0b0; stroke-width: 0.8; stroke-linecap: square\"/>\n","     </g>\n","     <g id=\"line2d_18\">\n","      <g>\n","       <use xlink:href=\"#m6ce9d77b77\" x=\"30.103125\" y=\"38.073893\" style=\"stroke: #000000; stroke-width: 0.8\"/>\n","      </g>\n","     </g>\n","     <g id=\"text_10\">\n","      <!-- 1.0 -->\n","      <g transform=\"translate(7.2 41.873112)scale(0.1 -0.1)\">\n","       <use xlink:href=\"#DejaVuSans-31\"/>\n","       <use xlink:href=\"#DejaVuSans-2e\" x=\"63.623047\"/>\n","       <use xlink:href=\"#DejaVuSans-30\" x=\"95.410156\"/>\n","      </g>\n","     </g>\n","    </g>\n","    <g id=\"ytick_5\">\n","     <g id=\"line2d_19\">\n","      <path d=\"M 30.103125 10.999219 \n","L 225.403125 10.999219 \n","\" clip-path=\"url(#p0bf9a0d35c)\" style=\"fill: none; stroke: #b0b0b0; stroke-width: 0.8; stroke-linecap: square\"/>\n","     </g>\n","     <g id=\"line2d_20\">\n","      <g>\n","       <use xlink:href=\"#m6ce9d77b77\" x=\"30.103125\" y=\"10.999219\" style=\"stroke: #000000; stroke-width: 0.8\"/>\n","      </g>\n","     </g>\n","     <g id=\"text_11\">\n","      <!-- 1.2 -->\n","      <g transform=\"translate(7.2 14.798437)scale(0.1 -0.1)\">\n","       <use xlink:href=\"#DejaVuSans-31\"/>\n","       <use xlink:href=\"#DejaVuSans-2e\" x=\"63.623047\"/>\n","       <use xlink:href=\"#DejaVuSans-32\" x=\"95.410156\"/>\n","      </g>\n","     </g>\n","    </g>\n","   </g>\n","   <g id=\"line2d_21\">\n","    <path d=\"M 12.743125 14.247209 \n","L 17.083125 42.784808 \n","L 21.423125 58.063591 \n","L 25.763125 67.399225 \n","L 30.103125 74.669143 \n","L 34.443125 102.952065 \n","L 38.783125 104.815126 \n","L 43.123125 106.974469 \n","L 47.463125 107.655153 \n","L 51.803125 108.726165 \n","L 56.143125 117.584499 \n","L 60.483125 116.31482 \n","L 64.823125 116.513468 \n","L 69.163125 117.045591 \n","L 73.503125 117.937739 \n","L 77.843125 120.804105 \n","L 82.183125 121.169415 \n","L 86.523125 121.033706 \n","L 90.863125 121.310544 \n","L 95.203125 122.133958 \n","L 99.543125 124.811447 \n","L 103.883125 126.726713 \n","L 108.223125 126.765673 \n","L 112.563125 126.998159 \n","L 116.903125 127.248849 \n","L 121.243125 129.939992 \n","L 125.583125 129.285594 \n","L 129.923125 129.505869 \n","L 134.263125 130.00989 \n","L 138.603125 130.233059 \n","L 142.943125 131.175296 \n","L 147.283125 132.257085 \n","L 151.623125 131.762244 \n","L 155.963125 131.687699 \n","L 160.303125 131.815547 \n","L 164.643125 133.782407 \n","L 168.983125 133.532075 \n","L 173.323125 133.978192 \n","L 177.663125 133.841758 \n","L 182.003125 134.184823 \n","L 186.343125 134.722969 \n","L 190.683125 135.255171 \n","L 195.023125 135.562529 \n","L 199.363125 135.261253 \n","L 203.703125 135.655755 \n","L 208.043125 137.792663 \n","L 212.383125 137.384312 \n","L 216.723125 137.449344 \n","L 221.063125 137.390509 \n","L 225.403125 137.059981 \n","\" clip-path=\"url(#p0bf9a0d35c)\" style=\"fill: none; stroke: #1f77b4; stroke-width: 1.5; stroke-linecap: square\"/>\n","   </g>\n","   <g id=\"line2d_22\">\n","    <path d=\"M 12.743125 93.01307 \n","L 17.083125 84.270956 \n","L 21.423125 79.301715 \n","L 25.763125 76.167343 \n","L 30.103125 73.598122 \n","L 34.443125 64.120216 \n","L 38.783125 63.062611 \n","L 43.123125 62.305036 \n","L 47.463125 61.923436 \n","L 51.803125 61.567942 \n","L 56.143125 58.235884 \n","L 60.483125 58.590294 \n","L 64.823125 58.622173 \n","L 69.163125 58.435592 \n","L 73.503125 58.179095 \n","L 77.843125 57.977109 \n","L 82.183125 57.369549 \n","L 86.523125 57.208283 \n","L 90.863125 57.091084 \n","L 95.203125 56.771212 \n","L 99.543125 56.221936 \n","L 103.883125 55.383728 \n","L 108.223125 55.291844 \n","L 112.563125 55.054633 \n","L 116.903125 55.040689 \n","L 121.243125 54.500516 \n","L 125.583125 54.326123 \n","L 129.923125 54.264242 \n","L 134.263125 54.129229 \n","L 138.603125 54.063744 \n","L 142.943125 53.93796 \n","L 147.283125 53.442911 \n","L 151.623125 53.484165 \n","L 155.963125 53.479477 \n","L 160.303125 53.438771 \n","L 164.643125 53.12788 \n","L 168.983125 52.942237 \n","L 173.323125 52.692837 \n","L 177.663125 52.635644 \n","L 182.003125 52.527257 \n","L 186.343125 52.239042 \n","L 190.683125 52.160284 \n","L 195.023125 52.11528 \n","L 199.363125 52.258732 \n","L 203.703125 52.143699 \n","L 208.043125 51.237693 \n","L 212.383125 51.434588 \n","L 216.723125 51.398959 \n","L 221.063125 51.400834 \n","L 225.403125 51.498419 \n","\" clip-path=\"url(#p0bf9a0d35c)\" style=\"fill: none; stroke-dasharray: 5.55,2.4; stroke-dashoffset: 0; stroke: #bf00bf; stroke-width: 1.5\"/>\n","   </g>\n","   <g id=\"line2d_23\">\n","    <path d=\"M 30.103125 71.280981 \n","L 51.803125 66.651212 \n","L 73.503125 64.72891 \n","L 95.203125 59.327512 \n","L 116.903125 57.960241 \n","L 138.603125 62.292189 \n","L 160.303125 62.84722 \n","L 182.003125 57.148001 \n","L 203.703125 63.023205 \n","L 225.403125 58.041465 \n","\" clip-path=\"url(#p0bf9a0d35c)\" style=\"fill: none; stroke-dasharray: 9.6,2.4,1.5,2.4; stroke-dashoffset: 0; stroke: #008000; stroke-width: 1.5\"/>\n","   </g>\n","   <g id=\"patch_3\">\n","    <path d=\"M 30.103125 143.969936 \n","L 30.103125 8.069936 \n","\" style=\"fill: none; stroke: #000000; stroke-width: 0.8; stroke-linejoin: miter; stroke-linecap: square\"/>\n","   </g>\n","   <g id=\"patch_4\">\n","    <path d=\"M 225.403125 143.969936 \n","L 225.403125 8.069936 \n","\" style=\"fill: none; stroke: #000000; stroke-width: 0.8; stroke-linejoin: miter; stroke-linecap: square\"/>\n","   </g>\n","   <g id=\"patch_5\">\n","    <path d=\"M 30.103125 143.969936 \n","L 225.403125 143.969936 \n","\" style=\"fill: none; stroke: #000000; stroke-width: 0.8; stroke-linejoin: miter; stroke-linecap: square\"/>\n","   </g>\n","   <g id=\"patch_6\">\n","    <path d=\"M 30.103125 8.069936 \n","L 225.403125 8.069936 \n","\" style=\"fill: none; stroke: #000000; stroke-width: 0.8; stroke-linejoin: miter; stroke-linecap: square\"/>\n","   </g>\n","   <g id=\"legend_1\">\n","    <g id=\"patch_7\">\n","     <path d=\"M 140.634375 99.537124 \n","L 218.403125 99.537124 \n","Q 220.403125 99.537124 220.403125 97.537124 \n","L 220.403125 54.502749 \n","Q 220.403125 52.502749 218.403125 52.502749 \n","L 140.634375 52.502749 \n","Q 138.634375 52.502749 138.634375 54.502749 \n","L 138.634375 97.537124 \n","Q 138.634375 99.537124 140.634375 99.537124 \n","z\n","\" style=\"fill: #ffffff; opacity: 0.8; stroke: #cccccc; stroke-linejoin: miter\"/>\n","    </g>\n","    <g id=\"line2d_24\">\n","     <path d=\"M 142.634375 60.601186 \n","L 152.634375 60.601186 \n","L 162.634375 60.601186 \n","\" style=\"fill: none; stroke: #1f77b4; stroke-width: 1.5; stroke-linecap: square\"/>\n","    </g>\n","    <g id=\"text_12\">\n","     <!-- train loss -->\n","     <g transform=\"translate(170.634375 64.101186)scale(0.1 -0.1)\">\n","      <defs>\n","       <path id=\"DejaVuSans-74\" d=\"M 1172 4494 \n","L 1172 3500 \n","L 2356 3500 \n","L 2356 3053 \n","L 1172 3053 \n","L 1172 1153 \n","Q 1172 725 1289 603 \n","Q 1406 481 1766 481 \n","L 2356 481 \n","L 2356 0 \n","L 1766 0 \n","Q 1100 0 847 248 \n","Q 594 497 594 1153 \n","L 594 3053 \n","L 172 3053 \n","L 172 3500 \n","L 594 3500 \n","L 594 4494 \n","L 1172 4494 \n","z\n","\" transform=\"scale(0.015625)\"/>\n","       <path id=\"DejaVuSans-72\" d=\"M 2631 2963 \n","Q 2534 3019 2420 3045 \n","Q 2306 3072 2169 3072 \n","Q 1681 3072 1420 2755 \n","Q 1159 2438 1159 1844 \n","L 1159 0 \n","L 581 0 \n","L 581 3500 \n","L 1159 3500 \n","L 1159 2956 \n","Q 1341 3275 1631 3429 \n","Q 1922 3584 2338 3584 \n","Q 2397 3584 2469 3576 \n","Q 2541 3569 2628 3553 \n","L 2631 2963 \n","z\n","\" transform=\"scale(0.015625)\"/>\n","       <path id=\"DejaVuSans-61\" d=\"M 2194 1759 \n","Q 1497 1759 1228 1600 \n","Q 959 1441 959 1056 \n","Q 959 750 1161 570 \n","Q 1363 391 1709 391 \n","Q 2188 391 2477 730 \n","Q 2766 1069 2766 1631 \n","L 2766 1759 \n","L 2194 1759 \n","z\n","M 3341 1997 \n","L 3341 0 \n","L 2766 0 \n","L 2766 531 \n","Q 2569 213 2275 61 \n","Q 1981 -91 1556 -91 \n","Q 1019 -91 701 211 \n","Q 384 513 384 1019 \n","Q 384 1609 779 1909 \n","Q 1175 2209 1959 2209 \n","L 2766 2209 \n","L 2766 2266 \n","Q 2766 2663 2505 2880 \n","Q 2244 3097 1772 3097 \n","Q 1472 3097 1187 3025 \n","Q 903 2953 641 2809 \n","L 641 3341 \n","Q 956 3463 1253 3523 \n","Q 1550 3584 1831 3584 \n","Q 2591 3584 2966 3190 \n","Q 3341 2797 3341 1997 \n","z\n","\" transform=\"scale(0.015625)\"/>\n","       <path id=\"DejaVuSans-69\" d=\"M 603 3500 \n","L 1178 3500 \n","L 1178 0 \n","L 603 0 \n","L 603 3500 \n","z\n","M 603 4863 \n","L 1178 4863 \n","L 1178 4134 \n","L 603 4134 \n","L 603 4863 \n","z\n","\" transform=\"scale(0.015625)\"/>\n","       <path id=\"DejaVuSans-6e\" d=\"M 3513 2113 \n","L 3513 0 \n","L 2938 0 \n","L 2938 2094 \n","Q 2938 2591 2744 2837 \n","Q 2550 3084 2163 3084 \n","Q 1697 3084 1428 2787 \n","Q 1159 2491 1159 1978 \n","L 1159 0 \n","L 581 0 \n","L 581 3500 \n","L 1159 3500 \n","L 1159 2956 \n","Q 1366 3272 1645 3428 \n","Q 1925 3584 2291 3584 \n","Q 2894 3584 3203 3211 \n","Q 3513 2838 3513 2113 \n","z\n","\" transform=\"scale(0.015625)\"/>\n","       <path id=\"DejaVuSans-20\" transform=\"scale(0.015625)\"/>\n","       <path id=\"DejaVuSans-6c\" d=\"M 603 4863 \n","L 1178 4863 \n","L 1178 0 \n","L 603 0 \n","L 603 4863 \n","z\n","\" transform=\"scale(0.015625)\"/>\n","       <path id=\"DejaVuSans-73\" d=\"M 2834 3397 \n","L 2834 2853 \n","Q 2591 2978 2328 3040 \n","Q 2066 3103 1784 3103 \n","Q 1356 3103 1142 2972 \n","Q 928 2841 928 2578 \n","Q 928 2378 1081 2264 \n","Q 1234 2150 1697 2047 \n","L 1894 2003 \n","Q 2506 1872 2764 1633 \n","Q 3022 1394 3022 966 \n","Q 3022 478 2636 193 \n","Q 2250 -91 1575 -91 \n","Q 1294 -91 989 -36 \n","Q 684 19 347 128 \n","L 347 722 \n","Q 666 556 975 473 \n","Q 1284 391 1588 391 \n","Q 1994 391 2212 530 \n","Q 2431 669 2431 922 \n","Q 2431 1156 2273 1281 \n","Q 2116 1406 1581 1522 \n","L 1381 1569 \n","Q 847 1681 609 1914 \n","Q 372 2147 372 2553 \n","Q 372 3047 722 3315 \n","Q 1072 3584 1716 3584 \n","Q 2034 3584 2315 3537 \n","Q 2597 3491 2834 3397 \n","z\n","\" transform=\"scale(0.015625)\"/>\n","      </defs>\n","      <use xlink:href=\"#DejaVuSans-74\"/>\n","      <use xlink:href=\"#DejaVuSans-72\" x=\"39.208984\"/>\n","      <use xlink:href=\"#DejaVuSans-61\" x=\"80.322266\"/>\n","      <use xlink:href=\"#DejaVuSans-69\" x=\"141.601562\"/>\n","      <use xlink:href=\"#DejaVuSans-6e\" x=\"169.384766\"/>\n","      <use xlink:href=\"#DejaVuSans-20\" x=\"232.763672\"/>\n","      <use xlink:href=\"#DejaVuSans-6c\" x=\"264.550781\"/>\n","      <use xlink:href=\"#DejaVuSans-6f\" x=\"292.333984\"/>\n","      <use xlink:href=\"#DejaVuSans-73\" x=\"353.515625\"/>\n","      <use xlink:href=\"#DejaVuSans-73\" x=\"405.615234\"/>\n","     </g>\n","    </g>\n","    <g id=\"line2d_25\">\n","     <path d=\"M 142.634375 75.279311 \n","L 152.634375 75.279311 \n","L 162.634375 75.279311 \n","\" style=\"fill: none; stroke-dasharray: 5.55,2.4; stroke-dashoffset: 0; stroke: #bf00bf; stroke-width: 1.5\"/>\n","    </g>\n","    <g id=\"text_13\">\n","     <!-- train acc -->\n","     <g transform=\"translate(170.634375 78.779311)scale(0.1 -0.1)\">\n","      <use xlink:href=\"#DejaVuSans-74\"/>\n","      <use xlink:href=\"#DejaVuSans-72\" x=\"39.208984\"/>\n","      <use xlink:href=\"#DejaVuSans-61\" x=\"80.322266\"/>\n","      <use xlink:href=\"#DejaVuSans-69\" x=\"141.601562\"/>\n","      <use xlink:href=\"#DejaVuSans-6e\" x=\"169.384766\"/>\n","      <use xlink:href=\"#DejaVuSans-20\" x=\"232.763672\"/>\n","      <use xlink:href=\"#DejaVuSans-61\" x=\"264.550781\"/>\n","      <use xlink:href=\"#DejaVuSans-63\" x=\"325.830078\"/>\n","      <use xlink:href=\"#DejaVuSans-63\" x=\"380.810547\"/>\n","     </g>\n","    </g>\n","    <g id=\"line2d_26\">\n","     <path d=\"M 142.634375 89.957436 \n","L 152.634375 89.957436 \n","L 162.634375 89.957436 \n","\" style=\"fill: none; stroke-dasharray: 9.6,2.4,1.5,2.4; stroke-dashoffset: 0; stroke: #008000; stroke-width: 1.5\"/>\n","    </g>\n","    <g id=\"text_14\">\n","     <!-- test acc -->\n","     <g transform=\"translate(170.634375 93.457436)scale(0.1 -0.1)\">\n","      <use xlink:href=\"#DejaVuSans-74\"/>\n","      <use xlink:href=\"#DejaVuSans-65\" x=\"39.208984\"/>\n","      <use xlink:href=\"#DejaVuSans-73\" x=\"100.732422\"/>\n","      <use xlink:href=\"#DejaVuSans-74\" x=\"152.832031\"/>\n","      <use xlink:href=\"#DejaVuSans-20\" x=\"192.041016\"/>\n","      <use xlink:href=\"#DejaVuSans-61\" x=\"223.828125\"/>\n","      <use xlink:href=\"#DejaVuSans-63\" x=\"285.107422\"/>\n","      <use xlink:href=\"#DejaVuSans-63\" x=\"340.087891\"/>\n","     </g>\n","    </g>\n","   </g>\n","  </g>\n"," </g>\n"," <defs>\n","  <clipPath id=\"p0bf9a0d35c\">\n","   <rect x=\"30.103125\" y=\"8.069936\" width=\"195.3\" height=\"135.9\"/>\n","  </clipPath>\n"," </defs>\n","</svg>\n"],"text/plain":["<Figure size 252x180 with 1 Axes>"]},"metadata":{"needs_background":"light"},"output_type":"display_data"}],"source":["d2l.train_ch6(net, train_iter, test_iter, num_epochs, lr, d2l.try_gpu())"],"id":"1fa34720"},{"cell_type":"markdown","metadata":{"origin_pos":25,"id":"403b1753"},"source":["## 争议\n","\n","直观地说，批量规范化被认为可以使优化更加平滑。\n","然而，我们必须小心区分直觉和对我们观察到的现象的真实解释。\n","回想一下，我们甚至不知道简单的神经网络（多层感知机和传统的卷积神经网络）为什么如此有效。\n","即使在暂退法和权重衰减的情况下，它们仍然非常灵活，因此无法通过常规的学习理论泛化保证来解释它们是否能够泛化到看不见的数据。\n","\n","在提出批量规范化的论文中，作者除了介绍了其应用，还解释了其原理：通过减少*内部协变量偏移*（internal covariate shift）。\n","据推测，作者所说的“内部协变量转移”类似于上述的投机直觉，即变量值的分布在训练过程中会发生变化。\n","然而，这种解释有两个问题：\n","1、这种偏移与严格定义的*协变量偏移*（covariate shift）非常不同，所以这个名字用词不当。\n","2、这种解释只提供了一种不明确的直觉，但留下了一个有待后续挖掘的问题：为什么这项技术如此有效？\n","本书旨在传达实践者用来发展深层神经网络的直觉。\n","然而，重要的是将这些指导性直觉与既定的科学事实区分开来。\n","最终，当你掌握了这些方法，并开始撰写自己的研究论文时，你会希望清楚地区分技术和直觉。\n","\n","随着批量规范化的普及，“内部协变量偏移”的解释反复出现在技术文献的辩论，特别是关于“如何展示机器学习研究”的更广泛的讨论中。\n","Ali Rahimi在接受2017年NeurIPS大会的“接受时间考验奖”（Test of Time Award）时发表了一篇令人难忘的演讲。他将“内部协变量转移”作为焦点，将现代深度学习的实践比作炼金术。\n","他对该示例进行了详细回顾 :cite:`Lipton.Steinhardt.2018`，概述了机器学习中令人不安的趋势。\n","此外，一些作者对批量规范化的成功提出了另一种解释：在某些方面，批量规范化的表现出与原始论文 :cite:`Santurkar.Tsipras.Ilyas.ea.2018`中声称的行为是相反的。\n","\n","然而，与机器学习文献中成千上万类似模糊的说法相比，内部协变量偏移没有更值得批评。\n","很可能，它作为这些辩论的焦点而产生共鸣，要归功于目标受众对它的广泛认可。\n","批量规范化已经被证明是一种不可或缺的方法。它适用于几乎所有图像分类器，并在学术界获得了数万引用。\n","\n","## 小结\n","\n","* 在模型训练过程中，批量规范化利用小批量的均值和标准差，不断调整神经网络的中间输出，使整个神经网络各层的中间输出值更加稳定。\n","* 批量规范化在全连接层和卷积层的使用略有不同。\n","* 批量规范化层和暂退层一样，在训练模式和预测模式下计算不同。\n","* 批量规范化有许多有益的副作用，主要是正则化。另一方面，”减少内部协变量偏移“的原始动机似乎不是一个有效的解释。\n","\n","## 练习\n","\n","1. 在使用批量规范化之前，我们是否可以从全连接层或卷积层中删除偏置参数？为什么？\n","1. 比较LeNet在使用和不使用批量规范化情况下的学习率。\n","    1. 绘制训练和测试准确度的提高。\n","    1. 你的学习率有多高？\n","1. 我们是否需要在每个层中进行批量规范化？尝试一下？\n","1. 你可以通过批量规范化来替换暂退法吗？行为会如何改变？\n","1. 确定参数`beta`和`gamma`，并观察和分析结果。\n","1. 查看高级API中有关`BatchNorm`的在线文档，以查看其他批量规范化的应用。\n","1. 研究思路：想想你可以应用的其他“规范化”转换？你可以应用概率积分变换吗？全秩协方差估计可以么？\n"],"id":"403b1753"},{"cell_type":"markdown","metadata":{"origin_pos":27,"tab":["pytorch"],"id":"ac8f8e6e"},"source":["[Discussions](https://discuss.d2l.ai/t/1874)\n"],"id":"ac8f8e6e"}],"metadata":{"accelerator":"GPU","kernelspec":{"display_name":"Python 3","name":"python3"},"language_info":{"name":"python"},"colab":{"provenance":[{"file_id":"https://github.com/d2l-ai/d2l-zh-pytorch-colab/blob/master/chapter_convolutional-modern/alexnet.ipynb","timestamp":1663767676144}],"toc_visible":true},"widgets":{"application/vnd.jupyter.widget-state+json":{"d5822e13bd314e51b6dee0f82cdec61f":{"model_module":"@jupyter-widgets/controls","model_name":"HBoxModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HBoxModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HBoxView","box_style":"","children":["IPY_MODEL_a7ad688df25b4d04a4d9c254433393cf","IPY_MODEL_f479f8d0e9794d098cf36181ea86ae49","IPY_MODEL_a7d34739a47049bf9311db55d0d601f1"],"layout":"IPY_MODEL_49f17da285724ae7886148ed10d84625"}},"a7ad688df25b4d04a4d9c254433393cf":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_e661161a211842abaac7bcf1e7147112","placeholder":"​","style":"IPY_MODEL_53df30ce3cf74550ab5798a04428fb6c","value":"100%"}},"f479f8d0e9794d098cf36181ea86ae49":{"model_module":"@jupyter-widgets/controls","model_name":"FloatProgressModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"FloatProgressModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"ProgressView","bar_style":"success","description":"","description_tooltip":null,"layout":"IPY_MODEL_1aee2a1bcd6741ac92acceb8386a8277","max":26421880,"min":0,"orientation":"horizontal","style":"IPY_MODEL_3661470a06df4c5ca5998fe03bf76643","value":26421880}},"a7d34739a47049bf9311db55d0d601f1":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_0ec39731080e4eec82d0616a4fdcf81f","placeholder":"​","style":"IPY_MODEL_98e33315f8404232b8d6a676bb17623a","value":" 26421880/26421880 [00:01&lt;00:00, 25956167.01it/s]"}},"49f17da285724ae7886148ed10d84625":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"e661161a211842abaac7bcf1e7147112":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"53df30ce3cf74550ab5798a04428fb6c":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"1aee2a1bcd6741ac92acceb8386a8277":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"3661470a06df4c5ca5998fe03bf76643":{"model_module":"@jupyter-widgets/controls","model_name":"ProgressStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"ProgressStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","bar_color":null,"description_width":""}},"0ec39731080e4eec82d0616a4fdcf81f":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"98e33315f8404232b8d6a676bb17623a":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"d2801dada8074b2395d512eaa91a148d":{"model_module":"@jupyter-widgets/controls","model_name":"HBoxModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HBoxModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HBoxView","box_style":"","children":["IPY_MODEL_325d3ef0ea954f339bf9986af34d836c","IPY_MODEL_3704aa63de714107be0aba2d505494f4","IPY_MODEL_fab456d0a0004eee825ece8b6cb9ad2f"],"layout":"IPY_MODEL_4b20325b29664dd5bed17caa47ac7326"}},"325d3ef0ea954f339bf9986af34d836c":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_62378cf9a46047e19c9a4c4e87a44c06","placeholder":"​","style":"IPY_MODEL_235ff7d308a84e96aa631ef07e4a1c8c","value":"100%"}},"3704aa63de714107be0aba2d505494f4":{"model_module":"@jupyter-widgets/controls","model_name":"FloatProgressModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"FloatProgressModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"ProgressView","bar_style":"success","description":"","description_tooltip":null,"layout":"IPY_MODEL_deaf95c84ff84f519aee7f69fc0a0508","max":29515,"min":0,"orientation":"horizontal","style":"IPY_MODEL_11269d1cef7942f6919519cc176b770d","value":29515}},"fab456d0a0004eee825ece8b6cb9ad2f":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_178e85fa7eb54940950d04c8c51c67c5","placeholder":"​","style":"IPY_MODEL_a2dcf75791514fdfa22fdf1a470b5b34","value":" 29515/29515 [00:00&lt;00:00, 271797.70it/s]"}},"4b20325b29664dd5bed17caa47ac7326":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"62378cf9a46047e19c9a4c4e87a44c06":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"235ff7d308a84e96aa631ef07e4a1c8c":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"deaf95c84ff84f519aee7f69fc0a0508":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"11269d1cef7942f6919519cc176b770d":{"model_module":"@jupyter-widgets/controls","model_name":"ProgressStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"ProgressStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","bar_color":null,"description_width":""}},"178e85fa7eb54940950d04c8c51c67c5":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"a2dcf75791514fdfa22fdf1a470b5b34":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"f9122b9104894b3eb5cb26add5bb6bc6":{"model_module":"@jupyter-widgets/controls","model_name":"HBoxModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HBoxModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HBoxView","box_style":"","children":["IPY_MODEL_e3c2a9ed2d094510b4b897b3e2d655d4","IPY_MODEL_abbead1db37e43a9ad4de89ea245cb1b","IPY_MODEL_e69eab9ed265451c908fe6bdb09ad92f"],"layout":"IPY_MODEL_eee1c312a7b2404bbea1a8d35caaacab"}},"e3c2a9ed2d094510b4b897b3e2d655d4":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_7988d1085b854da6834c44e4a0b10dab","placeholder":"​","style":"IPY_MODEL_cb8e7ab661664e06b1bd90db506aa872","value":"100%"}},"abbead1db37e43a9ad4de89ea245cb1b":{"model_module":"@jupyter-widgets/controls","model_name":"FloatProgressModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"FloatProgressModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"ProgressView","bar_style":"success","description":"","description_tooltip":null,"layout":"IPY_MODEL_735c2ebec0f648a9b1d695bd4aaf4439","max":4422102,"min":0,"orientation":"horizontal","style":"IPY_MODEL_f92d960292a54e63966f7deec79204ac","value":4422102}},"e69eab9ed265451c908fe6bdb09ad92f":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_669e619890104bd38f07e5c521d01ffe","placeholder":"​","style":"IPY_MODEL_7f6f3188f91049d8b905b94e4f8acff6","value":" 4422102/4422102 [00:00&lt;00:00, 8639747.34it/s]"}},"eee1c312a7b2404bbea1a8d35caaacab":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"7988d1085b854da6834c44e4a0b10dab":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"cb8e7ab661664e06b1bd90db506aa872":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"735c2ebec0f648a9b1d695bd4aaf4439":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"f92d960292a54e63966f7deec79204ac":{"model_module":"@jupyter-widgets/controls","model_name":"ProgressStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"ProgressStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","bar_color":null,"description_width":""}},"669e619890104bd38f07e5c521d01ffe":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"7f6f3188f91049d8b905b94e4f8acff6":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"1c88aff658e7429eab4387c443505863":{"model_module":"@jupyter-widgets/controls","model_name":"HBoxModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HBoxModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HBoxView","box_style":"","children":["IPY_MODEL_9f68a6adea7f40419b5c7bc864006399","IPY_MODEL_9dbbb787d6e045c9854c757b0d14aed8","IPY_MODEL_fd47a473f22a4127ba06f59be431951b"],"layout":"IPY_MODEL_017ed40d528b4724bd3911e635bf93fd"}},"9f68a6adea7f40419b5c7bc864006399":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_ccfae2cf427844bba2539c636c476193","placeholder":"​","style":"IPY_MODEL_8b1d0938a3024a93a2f5b52456e28c0f","value":"100%"}},"9dbbb787d6e045c9854c757b0d14aed8":{"model_module":"@jupyter-widgets/controls","model_name":"FloatProgressModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"FloatProgressModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"ProgressView","bar_style":"success","description":"","description_tooltip":null,"layout":"IPY_MODEL_7a2c2b565ed84942acde681fb6e6b328","max":5148,"min":0,"orientation":"horizontal","style":"IPY_MODEL_58dfd7d60f794cdf82a991794c10e91f","value":5148}},"fd47a473f22a4127ba06f59be431951b":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_568ccb51478244c1922d231f52c235a5","placeholder":"​","style":"IPY_MODEL_80211233886b43bdabcf0c853926efff","value":" 5148/5148 [00:00&lt;00:00, 171559.26it/s]"}},"017ed40d528b4724bd3911e635bf93fd":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"ccfae2cf427844bba2539c636c476193":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"8b1d0938a3024a93a2f5b52456e28c0f":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"7a2c2b565ed84942acde681fb6e6b328":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"58dfd7d60f794cdf82a991794c10e91f":{"model_module":"@jupyter-widgets/controls","model_name":"ProgressStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"ProgressStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","bar_color":null,"description_width":""}},"568ccb51478244c1922d231f52c235a5":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"80211233886b43bdabcf0c853926efff":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}}}}},"nbformat":4,"nbformat_minor":5}